Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
83 commits
Select commit Hold shift + click to select a range
2c5f4d8
style(frontend): enforce prettier via format:check script
ANUBprad Aug 23, 2026
8c74d4d
fix(evaluation): enforce tuple contracts in replay report value objects
ANUBprad Aug 23, 2026
a9f2b76
fix(evaluation): type redis trace repository and fix set slicing
ANUBprad Aug 23, 2026
e90d1ba
chore(deps): declare jsonschema and add types-jsonschema stubs
ANUBprad Aug 23, 2026
e3f4848
fix(metrics): align embedding metrics with real EmbeddingResponse con…
ANUBprad Aug 23, 2026
b3404a9
fix(api): wire replay endpoints to app DI and build explicit response…
ANUBprad Aug 23, 2026
4a67b04
chore(deps): add aiosqlite dev dependency for integration tests
ANUBprad Aug 23, 2026
1f019c7
feat(evaluation): register built-in metrics into production MetricEngine
ANUBprad Aug 23, 2026
b67c87f
feat(evaluation): execute selected metrics and persist results per item
ANUBprad Aug 23, 2026
5405e76
fix(evaluation): repair orchestration persistence stage to real repos…
ANUBprad Aug 23, 2026
25ec6da
test(evaluation): prove real workflow execution through provider, met…
ANUBprad Aug 23, 2026
565095a
feat(evaluation): establish explicit metric score contract
ANUBprad Aug 23, 2026
b1849ce
feat(providers): implement real EmbeddingProvider boundary on OpenAIP…
ANUBprad Aug 23, 2026
1c08626
feat(evaluation): wire embedding metrics to real provider boundary
ANUBprad Aug 23, 2026
7cfade1
feat(evaluation): harden LLM judges against fabricated scores
ANUBprad Aug 23, 2026
3f1b92b
test(evaluation): replace weak metric tests with known-answer tests
ANUBprad Aug 23, 2026
5d4afc8
feat(evaluation): record provider provenance in every metric result
ANUBprad Aug 23, 2026
9f837ed
test(evaluation): add eight canonical deterministic evaluation fixtures
ANUBprad Aug 23, 2026
2d44225
test(evaluation): cross-metric composition through the real pipeline
ANUBprad Aug 23, 2026
5a7a245
refactor(evaluation): remove duplicate 'relevance' metric
ANUBprad Aug 23, 2026
ff9d91c
perf(evaluation): reuse embeddings within a single item boundary
ANUBprad Aug 23, 2026
1990bb4
feat(evaluation): add reliability, reproducibility & benchmarking lay…
ANUBprad Aug 24, 2026
da768f0
feat(redteam): implement B.5 adaptive red team campaign engine
ANUBprad Aug 24, 2026
e4d271d
feat(b6): agent trajectory evaluation — domain model, tool execution,…
ANUBprad Aug 24, 2026
c9b3f05
fix(agents): wire AgentExecutor to use AgentLoop for real execution
ANUBprad Aug 24, 2026
1d6c5fd
fix(agents): wire Temporal workflow to execute AgentLoop via activity
ANUBprad Aug 24, 2026
70ec222
fix(trajectory): fix TrajectoryEvaluator conversation_history seriali…
ANUBprad Aug 24, 2026
0a60acf
cleanup: remove dead AudioProvider and VisionProvider contracts
ANUBprad Aug 24, 2026
d759a98
fix(otel): fix TracerProvider overwrite bug
ANUBprad Aug 24, 2026
54adaa2
fix(prometheus): add middleware to increment request metrics
ANUBprad Aug 24, 2026
bb130cd
feat(agents): add trajectory persistence layer
ANUBprad Aug 24, 2026
e589cfb
feat(evaluation): wire trace recording, provenance, fingerprint, and …
ANUBprad Aug 24, 2026
5c6dbf9
test(evaluation): prove verdict, trace, replay, and comparison integr…
ANUBprad Aug 24, 2026
c9f356d
fix(evaluation): fix mypy errors and ruff lint issues in workflow and…
ANUBprad Aug 24, 2026
cdcf919
fix(replay): load evaluation traces from persisted database records
ANUBprad Aug 24, 2026
9c38159
test(replay): add 15 integration tests for B.9.1.1 persistence gap
ANUBprad Aug 24, 2026
346fa42
fix(replay): clean up imports and formatting for B.9.1.1
ANUBprad Aug 24, 2026
710ec13
feat(redteam): add semantic effectiveness judge for attack evaluation
ANUBprad Aug 24, 2026
20e5b9e
test(redteam): add 20 tests for semantic effectiveness evaluation
ANUBprad Aug 24, 2026
93e829f
fix(redteam): formatting and lint fixes for semantic judge
ANUBprad Aug 24, 2026
355634d
fix(redteam): add evaluation_source and document INCONCLUSIVE/SCORE s…
ANUBprad Aug 24, 2026
1c812ea
feat(evaluation): add regression analysis domain and CLI
ANUBprad Aug 24, 2026
9e40537
test(evaluation): add 17 deterministic regression analysis tests
ANUBprad Aug 24, 2026
a554137
fix(evaluation): formatting and lint fixes for regression module
ANUBprad Aug 24, 2026
db635a4
fix(db): add agent trajectory migration
ANUBprad Aug 24, 2026
1cb39ba
fix(replay): enforce authentication on all trace endpoints
ANUBprad Aug 24, 2026
40b8781
fix(temporal): add heartbeats to long-running activities
ANUBprad Aug 24, 2026
e6279da
fix(temporal): make evaluation item execution retry-safe
ANUBprad Aug 24, 2026
1f06c50
fix(agents): make agent cancellation interrupt execution
ANUBprad Aug 24, 2026
5386a8d
fix(temporal): enforce workflow execution timeouts
ANUBprad Aug 24, 2026
1d43f07
fix(frontend): connect dashboard to real analytics data
ANUBprad Aug 24, 2026
d80febd
fix(config): remove dangerous secret defaults, fix lifecycle stop
ANUBprad Aug 24, 2026
6ca5097
fix(evaluation): use sa.delete() for mypy-compatible metric cleanup
ANUBprad Aug 24, 2026
610e46b
refactor(cleanup): remove dead execution modules
ANUBprad Aug 24, 2026
1598b6c
B.10.2 Phase 4b: Delete dead kernel modules
ANUBprad Aug 24, 2026
dc7ecec
B.10.2 Phase 4c: Delete dead provider runtime and selection modules
ANUBprad Aug 24, 2026
ed29940
B.10.2 Phase 4d: Delete dead frontend components
ANUBprad Aug 24, 2026
8a59706
B.10.2 Phase 5: Fix duplicate indexes
ANUBprad Aug 24, 2026
1a1bd7c
B.10.2 Phase 6: Remove dead test files for deleted modules
ANUBprad Aug 24, 2026
e75d53d
feat(api): add regression analysis endpoint and replay API methods
ANUBprad Aug 25, 2026
f3e2f43
feat(frontend): enhance run list and details with verdict, cancel, retry
ANUBprad Aug 25, 2026
e1ccb8c
feat(frontend): add replay viewer component for run details
ANUBprad Aug 25, 2026
b774ef0
feat(frontend): add run comparison page
ANUBprad Aug 25, 2026
d99dbc2
feat(frontend): add regression analysis component
ANUBprad Aug 25, 2026
e28e9e7
fix(redteam): remove mock data, wire cancel on attack run detail
ANUBprad Aug 25, 2026
416b01c
fix(frontend): resolve TypeScript errors in run pages
ANUBprad Aug 25, 2026
7494571
fix(backend): resolve all mypy strict type errors (28 errors in 6 files)
ANUBprad Aug 25, 2026
b364f30
fix(backend): resolve ruff format and lint errors in test and traject…
ANUBprad Aug 25, 2026
96b299d
fix(frontend): resolve prettier formatting issues for CI
ANUBprad Aug 25, 2026
40823bd
style(backend): apply ruff formatting to trajectory and agent files
ANUBprad Aug 25, 2026
fb6d4bf
fix(backend): register missing Temporal activities in DI container
ANUBprad Aug 25, 2026
27ebb1a
fix(backend): configure agent session factory and provider registry f…
ANUBprad Aug 25, 2026
a66fcf7
fix(backend): add explicit retry policies to Temporal workflow activi…
ANUBprad Aug 25, 2026
7f77827
test(backend): add deterministic tests for B.12.1 Temporal runtime re…
ANUBprad Aug 25, 2026
d87bdf8
style(backend): fix ruff formatting and import order in B.12.1 tests
ANUBprad Aug 25, 2026
809b063
fix(frontend): attach bearer token to API requests and correct SSE paths
ANUBprad Aug 25, 2026
ba0940f
test(frontend): add tests for API auth headers and SSE URL correctness
ANUBprad Aug 25, 2026
c3169dd
merge: resolve develop/main sync conflicts
ANUBprad Aug 25, 2026
6c575c4
fix(backend): remove stale mypy type: ignore comments
ANUBprad Aug 26, 2026
c16747f
fix(frontend): wrap useSearchParams in Suspense boundary for /runs/co…
ANUBprad Aug 26, 2026
b1f87e2
fix(backend): replace non-deterministic datetime.now() with workflow.…
ANUBprad Aug 26, 2026
709c531
style(backend): fix ruff formatting in integration test
ANUBprad Aug 26, 2026
d6dd55e
fix(backend): ensure APP_SECRET_KEY is set in test conftest for CI
ANUBprad Aug 26, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Add verdict, trace_data, provenance, and fingerprint to evaluation_runs.

Revision ID: 015
Revises: 014
Create Date: 2026-08-24
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import sqlalchemy as sa

from alembic import op

if TYPE_CHECKING:
from collections.abc import Sequence


# revision identifiers
revision: str = "015"
down_revision: str | None = "014"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
"""Add verdict, trace_data, provenance, and fingerprint columns."""
op.add_column(
"evaluation_runs",
sa.Column("verdict", sa.String(20), nullable=True),
)
op.add_column(
"evaluation_runs",
sa.Column("trace_data", sa.JSON, nullable=True),
)
op.add_column(
"evaluation_runs",
sa.Column("provenance", sa.JSON, nullable=True),
)
op.add_column(
"evaluation_runs",
sa.Column("fingerprint", sa.String(64), nullable=True),
)


def downgrade() -> None:
"""Remove verdict, trace_data, provenance, and fingerprint columns."""
op.drop_column("evaluation_runs", "fingerprint")
op.drop_column("evaluation_runs", "provenance")
op.drop_column("evaluation_runs", "trace_data")
op.drop_column("evaluation_runs", "verdict")
69 changes: 69 additions & 0 deletions backend/alembic/versions/016_create_agent_trajectories_table.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Create agent_trajectories table.

Revision ID: 016
Revises: 015
Create Date: 2026-08-24
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import sqlalchemy as sa

from alembic import op

if TYPE_CHECKING:
from collections.abc import Sequence


# revision identifiers
revision: str = "016"
down_revision: str | None = "015"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
"""Create the agent_trajectories table."""
op.create_table(
"agent_trajectories",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("run_id", sa.String(36), nullable=False),
sa.Column("agent_name", sa.String(255), nullable=False),
sa.Column("status", sa.String(30), nullable=False),
sa.Column("total_steps", sa.Integer(), server_default="0", nullable=False),
sa.Column("total_llm_calls", sa.Integer(), server_default="0", nullable=False),
sa.Column("total_tool_calls", sa.Integer(), server_default="0", nullable=False),
sa.Column("total_tokens_input", sa.Integer(), server_default="0", nullable=False),
sa.Column("total_tokens_output", sa.Integer(), server_default="0", nullable=False),
sa.Column("total_cost_usd", sa.Float(), server_default="0.0", nullable=False),
sa.Column("total_duration_ms", sa.Integer(), server_default="0", nullable=False),
sa.Column("final_response", sa.Text(), server_default="", nullable=False),
sa.Column("conversation_history", sa.JSON(), nullable=False),
sa.Column("tool_calls", sa.JSON(), nullable=False),
sa.Column("llm_calls", sa.JSON(), nullable=False),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("metadata", sa.JSON(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
op.create_index("ix_agent_trajectories_run_id", "agent_trajectories", ["run_id"])
op.create_index("ix_agent_trajectories_created_at", "agent_trajectories", ["created_at"])


def downgrade() -> None:
"""Drop the agent_trajectories table."""
op.drop_index("ix_agent_trajectories_created_at", table_name="agent_trajectories")
op.drop_index("ix_agent_trajectories_run_id", table_name="agent_trajectories")
op.drop_table("agent_trajectories")
16 changes: 13 additions & 3 deletions backend/app/agents/api/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

from datetime import timedelta
from typing import TYPE_CHECKING

from fastapi import APIRouter, Depends, HTTPException, Query, Request
Expand All @@ -25,7 +26,14 @@
RetryAgentRunHandler,
)
from app.agents.temporal.workflow import AgentRunWorkflow, AgentRunWorkflowInput
from app.core.dependencies import CurrentUser, get_current_user, get_db_session, get_temporal_client
from app.core.config import AppConfig
from app.core.dependencies import (
CurrentUser,
get_config_dependency,
get_current_user,
get_db_session,
get_temporal_client,
)
from app.infrastructure.database.repositories.agent_run_repository import (
SqlAlchemyAgentRunRepository,
)
Expand Down Expand Up @@ -114,6 +122,7 @@ async def create_agent_run(
current_user: CurrentUser = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
temporal_client: TemporalClient = Depends(get_temporal_client),
config: AppConfig = Depends(get_config_dependency),
) -> AgentRunResponse:
"""Create a new agent run and schedule its execution."""
repo = _get_repository(session)
Expand All @@ -140,10 +149,11 @@ async def create_agent_run(
AgentRunWorkflow.run,
AgentRunWorkflowInput(
run_id=str(run.id),
total_steps=body.max_steps,
max_steps=body.max_steps,
),
id=workflow_id,
task_queue="redops-agents",
task_queue=config.temporal_task_queue,
execution_timeout=timedelta(hours=1),
)

queue_handler = QueueAgentRunHandler(repo)
Expand Down
52 changes: 52 additions & 0 deletions backend/app/agents/domain/contracts/agent_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,55 @@ async def publish(self, event: object) -> None:
async def publish_many(self, events: list[object]) -> None:
"""Publish multiple domain events."""
...


@dataclass
class TrajectoryQuery:
"""Query parameters for listing trajectories."""

run_id: str | None = None
agent_name: str | None = None
status: str | None = None
sort_by: str = "created_at"
sort_order: str = "desc"
page: int = 1
page_size: int = 20


@dataclass
class PaginatedTrajectories:
"""Paginated result for trajectory listing."""

items: list[dict[str, object]] = field(default_factory=list)
total: int = 0
page: int = 1
page_size: int = 20


class AgentTrajectoryRepository(ABC):
"""Repository for agent trajectory persistence."""

@abstractmethod
async def save(self, trajectory: dict[str, object]) -> None:
"""Save a trajectory."""
...

@abstractmethod
async def find_by_id(self, trajectory_id: str) -> dict[str, object] | None:
"""Find a trajectory by ID."""
...

@abstractmethod
async def find_by_run_id(self, run_id: str) -> dict[str, object] | None:
"""Find trajectory by run ID."""
...

@abstractmethod
async def list(self, query: TrajectoryQuery) -> PaginatedTrajectories:
"""List trajectories with filtering and pagination."""
...

@abstractmethod
async def delete(self, trajectory_id: str) -> bool:
"""Delete a trajectory by ID."""
...
Loading
Loading