Add langgraph observability and layers (#6)

* add langchain callback

* implement layers client and nodes

* ignore non-code paths on workflow triggers; move layers module

---------

Co-authored-by: akhatre <akhatre@users.noreply.github.com>

Author: akhatre

.github/workflows/pr-checks.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - main
     types: [opened, synchronize, reopened]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - 'README.md'
 
 jobs:
   test_and_check_version:

.github/workflows/publish.yml

@@ -4,6 +4,10 @@ on:
   push:
     branches:
       - main
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - 'README.md'
 
 jobs:
   deploy:

Makefile

@@ -3,4 +3,9 @@
 .PHONY: test-publish
 test-publish:
 	poetry build
-	twine upload --repository testpypi dist/* --
+	twine upload --repository testpypi dist/* --
+
+
+.PHONY: format
+format:
+	poetry run ruff format

overmind/agents.py

@@ -29,7 +29,7 @@ def create(
     ) -> Dict[str, str]:
         """
         Create a new agent.
-        
+
         Args:
             agent_id: Unique identifier for the agent
             agent_model: The AI model to use (e.g., 'gpt-4o')
@@ -71,7 +71,7 @@ def list(self) -> List[AgentResponse]:
     def get(self, agent_id: str) -> AgentResponse:
         """
         Get a specific agent by ID.
-        
+
         Args:
             agent_id: The unique identifier of the agent to retrieve
         """
@@ -92,7 +92,7 @@ def update(
     ) -> Dict[str, str]:
         """
         Update an existing agent.
-        
+
         Args:
             agent_id: Unique identifier for the agent
             agent_model: The AI model to use (e.g., 'gpt-4o')
@@ -128,7 +128,7 @@ def update(
     def delete(self, agent_id: str) -> Dict[str, str]:
         """
         Delete an agent by ID.
-        
+
         Args:
             agent_id: The unique identifier of the agent to delete
         """

overmind/client.py

@@ -3,17 +3,18 @@
 """
 
 import os
-from typing import Any, Dict, List, Optional, Union
+from functools import lru_cache
+from typing import Any, Dict, List, Optional, Sequence, Union
 from urllib.parse import urljoin
 
 import requests
 
 from .exceptions import OvermindAPIError, OvermindAuthenticationError, OvermindError
-from .models import InvocationResponse
+from .models import InvocationResponse, LayerResponse
 from .agents import AgentsClient
 from .policies import PoliciesClient
 from .invocations import InvocationsClient
-
+from .utils.api_settings import get_api_settings
 
 # Mapping of common environment variables to provider parameter names
 COMMON_ENV_VARS = {
@@ -96,24 +97,9 @@ def __init__(
         Raises:
             OvermindError: If no API key is provided and OVERMIND_API_KEY environment variable is not set
         """
-        # Get API key from parameter or environment variable
-        if overmind_api_key is None:
-            overmind_api_key = os.getenv("OVERMIND_API_KEY")
-            if overmind_api_key is None:
-                raise OvermindError(
-                    "No Overmind API key provided. Either pass 'overmind_api_key' parameter "
-                    "or set OVERMIND_API_KEY environment variable."
-                )
-
-        if base_url is None:
-            base_url = os.getenv("OVERMIND_API_URL")
-            if base_url is None:
-                base_url = (
-                    "https://api.evallab.dev"
-                )
-
-        self.overmind_api_key = overmind_api_key
-        self.base_url = base_url.rstrip("/")
+        self.overmind_api_key, self.base_url, self.traces_base_url = get_api_settings(
+            overmind_api_key, base_url, None
+        )
 
         # Start with provided provider parameters
         self.provider_parameters = (
@@ -129,7 +115,7 @@ def __init__(
         self.session = requests.Session()
         self.session.headers.update(
             {
-                "Authorization": f"Bearer {overmind_api_key}",
+                "Authorization": f"Bearer {self.overmind_api_key}",
                 "Content-Type": "application/json",
             }
         )
@@ -237,3 +223,55 @@ def invoke(
         )
 
         return InvocationResponse(**response_data)
+
+
+class OvermindLayersClient:
+    def __init__(
+        self,
+        overmind_api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        traces_base_url: Optional[str] = None,
+    ):
+        self.overmind_api_key, self.base_url, self.traces_base_url = get_api_settings(
+            overmind_api_key, base_url, traces_base_url
+        )
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "Authorization": f"Bearer {self.overmind_api_key}",
+                "Content-Type": "application/json",
+            }
+        )
+
+    def run_layer(
+        self, input_data: str, policies: Sequence[str | dict]
+    ) -> LayerResponse:
+        """
+        Run a layer of the Overmind API.
+        """
+        payload = {
+            "input_data": input_data,
+            "policies": policies,
+        }
+
+        response_data = self.session.request(
+            "POST", f"{self.base_url}/api/v1/layers/run", json=payload
+        )
+
+        if response_data.status_code != 200:
+            raise OvermindAPIError(
+                message=response_data.text,
+                status_code=response_data.status_code,
+                response_data=response_data.json(),
+            )
+
+        return LayerResponse(**response_data.json())
+
+
+@lru_cache
+def get_layers_client(
+    overmind_api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    traces_base_url: Optional[str] = None,
+):
+    return OvermindLayersClient(overmind_api_key, base_url, traces_base_url)

overmind/formatters.py

@@ -10,55 +10,75 @@ def summarise_invocation(result) -> None:
     # -- 2. Set up Rich console and formatting maps --
     console = Console()
     res_map = {
-        'passed': {'color': 'green', 'emoji': '✅'},
-        'altered': {'color': 'yellow', 'emoji': '⚠️'},
-        'rejected': {'color': 'red', 'emoji': '❌'},
+        "passed": {"color": "green", "emoji": "✅"},
+        "altered": {"color": "yellow", "emoji": "⚠️"},
+        "rejected": {"color": "red", "emoji": "❌"},
     }
 
-
     # -- 3. Build the Rich renderable components --
 
     # A Table for Invocation Results
-    inv_table = Table(show_header=True, header_style="bold white", title="Invocation Results")
+    inv_table = Table(
+        show_header=True, header_style="bold white", title="Invocation Results"
+    )
     inv_table.add_column("Component", style="dim")
     inv_table.add_column("Outcome", justify="center")
 
     for part, outcome_str in result.invocation_results.items():
-        style = res_map.get(outcome_str, {}).get('color', 'white')
-        emoji = res_map.get(outcome_str, {}).get('emoji', '➡️')
+        style = res_map.get(outcome_str, {}).get("color", "white")
+        emoji = res_map.get(outcome_str, {}).get("emoji", "➡️")
         outcome_text = Text(f"{emoji} {outcome_str.upper()}", style=style)
         inv_table.add_row(part.replace("_", " ").capitalize(), outcome_text)
 
     # A Table for Policy Results
-    pol_table = Table(show_header=True, header_style="bold white", title="\nPolicy Results")
+    pol_table = Table(
+        show_header=True, header_style="bold white", title="\nPolicy Results"
+    )
     pol_table.add_column("Policy", style="dim", width=30)
     pol_table.add_column("Details", width=70)
 
     for policy, outcome in result.policy_results.items():
         # Assuming the outcome is a dict, format it nicely with Syntax
         outcome_str = json.dumps(outcome, indent=2)
-        outcome_syntax = Syntax(outcome_str, "json", theme="solarized-dark", word_wrap=True)
+        outcome_syntax = Syntax(
+            outcome_str, "json", theme="solarized-dark", word_wrap=True
+        )
         pol_table.add_row(policy, outcome_syntax)
 
     # Syntax Panels for Processed Input and Output
-    input_color = res_map.get(result.invocation_results['input'], {}).get('color', 'white')
-    output_color = res_map.get(result.invocation_results.get('output'), {}).get('color', 'white')
-    
+    input_color = res_map.get(result.invocation_results["input"], {}).get(
+        "color", "white"
+    )
+    output_color = res_map.get(result.invocation_results.get("output"), {}).get(
+        "color", "white"
+    )
+
     input_panel = Panel(
-        Syntax(result.processed_input, "plain", theme="solarized-dark", line_numbers=False, word_wrap=True),
+        Syntax(
+            result.processed_input,
+            "plain",
+            theme="solarized-dark",
+            line_numbers=False,
+            word_wrap=True,
+        ),
         title="[bold]Processed Input[/bold]",
-        border_style=input_color, # The border color reflects the outcome!
+        border_style=input_color,  # The border color reflects the outcome!
         title_align="left",
     )
 
     output_panel = Panel(
-        Syntax((result.processed_output or result.raw_output)[:2000], "plain", theme="solarized-dark", line_numbers=False, word_wrap=True),
+        Syntax(
+            (result.processed_output or result.raw_output)[:2000],
+            "plain",
+            theme="solarized-dark",
+            line_numbers=False,
+            word_wrap=True,
+        ),
         title="[bold]Processed Output[/bold]",
-        border_style=output_color, # The border color reflects the outcome!
+        border_style=output_color,  # The border color reflects the outcome!
         title_align="left",
     )
 
-
     # -- 4. Group components and print inside a final Panel --
     final_group = Group(
         inv_table,
@@ -71,6 +91,6 @@ def summarise_invocation(result) -> None:
         Panel(
             final_group,
             title="[bold cyan]Invocation & Policy Summary[/bold cyan]",
-            border_style="green"
+            border_style="green",
         )
-    )
+    )