feat: end-to-end WhatsApp death-certificate verification

agents/poc_deathCertParserAgent.yaml

@@ -16,13 +16,17 @@ system_prompt: |
   You must track the number of times you have asked the user to provide the death certificate. If you ask for the document 3 times and the user still has not provided one, you must halt all normal conversational responses. From that point forward, reply to EVERY message from the user exclusively with this exact phrase: "I'm sorry, but I cannot proceed without proof". Continue doing this until a document is finally uploaded.
 
   PHASE 3: DOCUMENT PROCESSING & TOOL EXECUTION
-  The moment the user provides an image or PDF, you must immediately notify them by saying: "Your request is being processed." 
-  Following that notification, review the document (which may be in French or Arabic), extract all written text, translate it into English, and immediately pass that data into the `death_certificate_consistency` tool.
+  The moment the user provides an image or PDF, your VERY FIRST action must be to call the `death_certificate_verification`
+  tool — call it before writing any reply. Do NOT respond with only "Your request is being processed" and stop; a standalone
+  acknowledgement without calling the tool is a failure. You do not pass the document in — the tool automatically retrieves
+  the user's most recent upload, verifies it, and forwards the case to GiveLight when it passes.
+  Only after the tool returns do you write your reply, and it must reflect the tool's outcome: if it was forwarded, reassure
+  them their request is now with GiveLight; if it needs review or a clearer image, gently explain the next step.
 
 skills:
-  # TBD 
+  # TBD
 tools:
-  - death_certificate_consistency
+  - death_certificate_verification
 
 provider:
   type: gemini

src/api.py

@@ -8,43 +8,70 @@
 
 import logging
 import os
+from dataclasses import dataclass
 from typing import Any
 
 from fastapi import FastAPI, Header, HTTPException, Request
 from starlette.concurrency import run_in_threadpool
 
 from .chat import chat
+from .whatsapp_media import download_media
 
 logger = logging.getLogger(__name__)
 
 app = FastAPI(title="b2 WhatsApp Adapter")
 
 _WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET", "")
 
+# WhatsApp media message types that carry a downloadable media ID.
+_MEDIA_TYPES = ("image", "document")
 
-def _extract_message(payload: dict[str, Any]) -> tuple[str | None, str | None]:
-    """Return (wa_id, text) for a text message, or (None, None) for anything else."""
+
+@dataclass
+class InboundMessage:
+    """A normalized inbound WhatsApp message."""
+
+    wa_id: str | None = None
+    text: str | None = None
+    media_id: str | None = None
+    mime_type: str | None = None
+
+
+def _extract_message(payload: dict[str, Any]) -> InboundMessage:
+    """Normalize a WhatsApp webhook payload into an InboundMessage.
+
+    Handles text and media (image/document) messages; returns an empty
+    InboundMessage for anything else (e.g. status updates).
+    """
     try:
         entry = (payload.get("entry") or [])[0]
         value = (entry.get("changes") or [])[0].get("value", {})
         contacts = value.get("contacts") or []
         messages = value.get("messages") or []
 
         if not messages:
-            return None, None
+            return InboundMessage()
 
         msg = messages[0]
-        if msg.get("type") != "text":
-            return None, None
-
+        msg_type = msg.get("type")
         wa_id = (contacts[0].get("wa_id") if contacts else None) or msg.get("from")
-        text = (msg.get("text") or {}).get("body", "").strip()
-        if not text:
-            return None, None
 
-        return wa_id, text
+        if msg_type == "text":
+            text = (msg.get("text") or {}).get("body", "").strip()
+            if not text:
+                return InboundMessage(wa_id=wa_id)
+            return InboundMessage(wa_id=wa_id, text=text)
+
+        if msg_type in _MEDIA_TYPES:
+            media = msg.get(msg_type) or {}
+            media_id = media.get("id")
+            if not media_id:
+                return InboundMessage(wa_id=wa_id)
+            return InboundMessage(wa_id=wa_id, media_id=media_id, mime_type=media.get("mime_type"))
+
+        return InboundMessage(wa_id=wa_id)
     except (IndexError, AttributeError, TypeError, KeyError):
-        return None, None
+        return InboundMessage()
 
 
 @app.post("/message")
@@ -58,15 +85,32 @@ async def message_endpoint(
     payload = await request.json()
     logger.info("api.message received object=%s", payload.get("object"))
 
-    wa_id, text = _extract_message(payload)
-    if text is None:
-        logger.info("api.message skipped — no text payload")
-        return {"response": None}
-
-    logger.info("api.message routing wa_id=%s chars=%d", wa_id, len(text))
-    response = await run_in_threadpool(chat, text=text, session_id=wa_id)
-    logger.info("api.message done wa_id=%s response_chars=%d", wa_id, len(response))
-    return {"response": response}
+    message = _extract_message(payload)
+
+    if message.text is not None:
+        logger.info("api.message routing text wa_id=%s chars=%d", message.wa_id, len(message.text))
+        response = await run_in_threadpool(chat, text=message.text, session_id=message.wa_id)
+        logger.info("api.message done wa_id=%s response_chars=%d", message.wa_id, len(response))
+        return {"response": response}
+
+    if message.media_id is not None:
+        logger.info("api.message media wa_id=%s media=%s", message.wa_id, message.media_id)
+        media = await download_media(message.media_id)
+        if media is None:
+            logger.info("api.message skipped — media download unavailable")
+            return {"response": None}
+        image_bytes, mime_type = media
+        response = await run_in_threadpool(
+            chat,
+            image_bytes=image_bytes,
+            image_media_type=mime_type,
+            session_id=message.wa_id,
+        )
+        logger.info("api.message done wa_id=%s response_chars=%d", message.wa_id, len(response))
+        return {"response": response}
+
+    logger.info("api.message skipped — no actionable payload")
+    return {"response": None}
 
 
 @app.get("/health")

src/chat.py

@@ -3,8 +3,17 @@
 from collections.abc import Sequence
 
 from dotenv import load_dotenv
-from pydantic_ai.messages import BinaryImage, ImageUrl, TextContent, UserContent
-
+from pydantic_ai.messages import (
+    BinaryImage,
+    ImageUrl,
+    ModelMessage,
+    TextContent,
+    TextPart,
+    UserContent,
+    UserPromptPart,
+)
+
+from .orchestrator.context import SessionContext
 from .router import AgentRouter
 from .session import Session
 from .session_store import FirestoreSessionStore
@@ -13,11 +22,29 @@
 
 DEFAULT_IMAGE_MEDIA_TYPE = "image/jpeg"
 IMAGE_PROMPT_TEXT = "The user sent this image on WhatsApp. Analyze it and provide a helpful response."
-IMAGE_ROUTING_TEXT = "A WhatsApp user sent an image and needs help interpreting or responding to it."
+IMAGE_ROUTING_TEXT = "A WhatsApp user uploaded a document image such as a death certificate for verification."
+# Text notice given to the model when an image arrives. The image bytes go to the
+# transient store (for tools to pull), never into the model's prompt.
+IMAGE_ARRIVED_PROMPT = "The user has just uploaded a document image."
 
 _router: AgentRouter | None = None
 
 
+def _render_history_text(history: Sequence[ModelMessage] | None) -> str:
+    """Flatten prior user/assistant turns into plain text for use as a narrative."""
+    lines: list[str] = []
+    for message in history or []:
+        for part in getattr(message, "parts", []):
+            content = getattr(part, "content", None)
+            if not isinstance(content, str) or not content.strip():
+                continue
+            if isinstance(part, UserPromptPart):
+                lines.append(f"user: {content.strip()}")
+            elif isinstance(part, TextPart):
+                lines.append(f"assistant: {content.strip()}")
+    return "\n".join(lines)
+
+
 def _get_router() -> AgentRouter:
     global _router
     if _router is None:
@@ -36,19 +63,24 @@ def chat(
 ) -> str:
     """Route one WhatsApp text or image message through the agent loop and return the response."""
 
-    prompt = _build_prompt(
+    store = FirestoreSessionStore() if session_id is not None else None
+    history = store.load_history(session_id) if store is not None else None
+    history_text = _render_history_text(history)
+
+    prompt, route_query = _prepare_turn(
         text=text,
         image_bytes=image_bytes,
         image_url=image_url,
         image_media_type=image_media_type,
+        history_text=history_text,
+        store=store,
+        session_id=session_id,
     )
-    route_query = text.strip() if text is not None else IMAGE_ROUTING_TEXT
 
     router = _get_router()
     agent, _metadata = router.route_with_metadata(route_query)
-    store = FirestoreSessionStore() if session_id is not None else None
-    history = store.load_history(session_id) if store is not None else None
-    session = Session(agent, history=history)
+    deps = SessionContext(session_id=session_id, store=store, history_text=history_text)
+    session = Session(agent, history=history, deps=deps)
 
     response = "".join(session.send_stream(prompt))
     if store is not None:
@@ -61,6 +93,42 @@ def chat(
     return response
 
 
+def _prepare_turn(
+    *,
+    text: str | None,
+    image_bytes: bytes | None,
+    image_url: str | None,
+    image_media_type: str,
+    history_text: str,
+    store: FirestoreSessionStore | None,
+    session_id: str | None,
+) -> tuple[str | Sequence[UserContent], str]:
+    """Return (model_prompt, routing_query) and persist any uploaded image.
+
+    An uploaded image (image_bytes) is saved to the transient store and the model
+    is handed only a text notice — the bytes are pulled later by a context-aware
+    tool and never enter the model's prompt.
+    """
+    supplied = sum(value is not None for value in (text, image_bytes, image_url))
+    if supplied != 1:
+        raise ValueError("Provide exactly one of text, image_bytes, or image_url")
+
+    if image_bytes is not None:
+        if not image_bytes:
+            raise ValueError("image_bytes must not be empty")
+        if store is not None and session_id is not None:
+            store.save_media(session_id, image_bytes, mime_type=image_media_type)
+        return IMAGE_ARRIVED_PROMPT, (history_text or IMAGE_ROUTING_TEXT)
+
+    if text is not None:
+        prompt = _build_prompt(text=text, image_bytes=None, image_url=None, image_media_type=image_media_type)
+        return prompt, text.strip()
+
+    # image_url — legacy path: the referenced image is passed to the model directly.
+    prompt = _build_prompt(text=None, image_bytes=None, image_url=image_url, image_media_type=image_media_type)
+    return prompt, IMAGE_ROUTING_TEXT
+
+
 def _build_prompt(
     *,
     text: str | None,