From 9e76b3fed1e0e29947273d5b7288e6eef3ddd3ad Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 27 Apr 2026 11:00:46 -0300 Subject: [PATCH 01/70] feat(core): add flat agentops.yaml schema and evaluator inference Foundation for the 1.0 revamp (#107). Adds two new modules alongside the existing layered config system; nothing wired up to the runner yet. - core/agentops_config.py: flat AgentOpsConfig (Pydantic v2) with version, agent, dataset, thresholds, optional protocol and HTTP fields, optional evaluators override. classify_agent() resolves the agent value into one of foundry_prompt, foundry_hosted, http_json, model_direct. Rejects legacy keys (target/bundle/scenario/run/etc) with a clear error. - core/evaluators.py: catalog of ~12 evaluator presets aligned with the azure-ai-evaluation SDK categories. select_evaluators() picks the set from (target_kind, dataset_shape): always baseline quality (Coherence, Fluency, Similarity, F1Score); +RAG when rows have context; +Agent evaluators when rows have tool_calls/tool_definitions. Model-direct targets stay quality-only. Optional overrides bypass inference. - detect_dataset_shape() inspects a JSONL dataset and reports which optional fields are populated. - merge_thresholds() combines preset defaults with user overrides. 53 unit tests added; full suite still green (319 passed, 1 skipped). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/core/agentops_config.py | 421 +++++++++++++++++++++++++++ src/agentops/core/evaluators.py | 405 ++++++++++++++++++++++++++ tests/unit/test_agentops_config.py | 227 +++++++++++++++ tests/unit/test_evaluators.py | 226 ++++++++++++++ 4 files changed, 1279 insertions(+) create mode 100644 src/agentops/core/agentops_config.py create mode 100644 src/agentops/core/evaluators.py create mode 100644 tests/unit/test_agentops_config.py create mode 100644 tests/unit/test_evaluators.py diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py new file mode 100644 index 00000000..61bdfd18 --- /dev/null +++ b/src/agentops/core/agentops_config.py @@ -0,0 +1,421 @@ +"""Flat ``agentops.yaml`` schema for AgentOps 1.0. + +This module defines the user-facing configuration shape that replaces the +layered ``run.yaml`` + ``bundle.yaml`` + ``dataset.yaml`` files of pre-1.0 +AgentOps. + +Design goals: + +* One file. ``agentops.yaml`` is the single source of truth. +* No ``scenario`` field. The toolkit derives the target type from the + ``agent`` value and the evaluator set from the dataset row shape (see + :mod:`agentops.core.evaluators`). +* No bundle / dataset YAML configs. Datasets are plain JSONL files referenced + directly by path. + +The minimal valid config is three lines:: + + version: 1 + agent: my-rag-agent:3 + dataset: ./qa.jsonl + +The :func:`classify_agent` helper resolves ``agent`` into one of four target +kinds — ``foundry_prompt``, ``foundry_hosted``, ``http_json``, or +``model_direct`` — based on the value shape and optional ``protocol`` field. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +# --------------------------------------------------------------------------- +# Public type aliases +# --------------------------------------------------------------------------- + +#: Wire protocol for hosted / HTTP targets. +Protocol = Literal["responses", "invocations", "http-json"] + +#: How thresholds compare against measured metric values. +Criteria = Literal[">=", ">", "<=", "<", "==", "true", "false"] + +#: Resolved target kind. Derived from the ``agent`` value, never set by the user. +TargetKind = Literal[ + "foundry_prompt", # name:version + "foundry_hosted", # https://...foundry... endpoint + "http_json", # any other https URL + "model_direct", # model: +] + + +# --------------------------------------------------------------------------- +# Threshold model +# --------------------------------------------------------------------------- + + +class Threshold(BaseModel): + """A pass/fail rule for a single metric. + + Users typically write thresholds as a dict keyed by metric name in + ``agentops.yaml``:: + + thresholds: + groundedness: ">=3" + coherence: ">=3" + avg_latency_seconds: "<=10" + + Each value is parsed by :meth:`from_expression` into a ``Threshold``. + """ + + metric: str + criteria: Criteria + value: Optional[float] = None + + model_config = ConfigDict(frozen=True) + + @classmethod + def from_expression(cls, metric: str, expression: Any) -> "Threshold": + """Parse a shorthand string like ``">=3"`` or a bool like ``true``.""" + if isinstance(expression, bool): + return cls(metric=metric, criteria="true" if expression else "false") + if isinstance(expression, (int, float)): + return cls(metric=metric, criteria=">=", value=float(expression)) + if not isinstance(expression, str): + raise ValueError( + f"threshold for {metric!r} must be a string, number, or bool" + ) + text = expression.strip() + if text.lower() in {"true", "false"}: + return cls(metric=metric, criteria=text.lower()) # type: ignore[arg-type] + for op in (">=", "<=", "==", ">", "<"): + if text.startswith(op): + rest = text[len(op):].strip() + try: + return cls(metric=metric, criteria=op, value=float(rest)) # type: ignore[arg-type] + except ValueError as exc: + raise ValueError( + f"threshold for {metric!r}: cannot parse number from {text!r}" + ) from exc + raise ValueError( + f"threshold for {metric!r}: expected '>=N', '<=N', '>N', ' str: + if not value.strip(): + raise ValueError("evaluator name must be non-empty") + return value + + +# --------------------------------------------------------------------------- +# Top-level config +# --------------------------------------------------------------------------- + + +_LEGACY_TOP_LEVEL_KEYS = { + "target", + "bundle", + "execution", + "output", + "scenario", + "backend", + "run", +} + + +class AgentOpsConfig(BaseModel): + """Top-level ``agentops.yaml`` model. + + Fields: + + ``version`` + Schema version. Must be ``1`` in this release. + + ``agent`` + The thing under evaluation. One of: + + * ``":"`` — a Foundry prompt agent (e.g. ``"my-rag:3"``). + * ``"https://..."`` — a Foundry hosted endpoint or any HTTP/JSON agent. + * ``"model:"`` — a Foundry model deployment (raw model). + + See :func:`classify_agent` for the full resolution table. + + ``dataset`` + Relative path to a JSONL file with one evaluation row per line. Rows + must contain at least ``input`` and ``expected``; optional fields + ``context``, ``tool_calls``, and ``tool_definitions`` drive evaluator + auto-selection. + + ``thresholds`` + Optional dict of metric name → criteria expression. When omitted, the + evaluator catalog provides sensible defaults per metric. + + ``protocol`` + Optional, only relevant for URL-based ``agent`` values. Defaults to + ``"responses"`` for Foundry hosted endpoints and ``"http-json"`` for + any other HTTPS URL. + + ``request_field`` / ``response_field`` / ``tool_calls_field`` + ``http-json`` and ``invocations`` only. JSON keys / dot-paths used to + marshal each dataset row into the request body and to extract the + response. Defaults are sensible for OpenAI-compatible / ACA endpoints. + + ``headers`` / ``auth_header_env`` + Optional HTTP request configuration for ``http-json`` and + ``invocations`` targets. + + ``evaluators`` + Optional escape hatch: explicit list of evaluator names that overrides + the auto-selection rules. Most users should leave this unset. + """ + + version: int = Field(..., description="Schema version. Must be 1.") + agent: str = Field(..., description="Target identifier (name:version, URL, or model:deployment)") + dataset: Path = Field(..., description="Path to a JSONL dataset file") + + thresholds: Dict[str, Any] = Field( + default_factory=dict, + description="Metric name -> criteria expression (e.g. '>=3').", + ) + + protocol: Optional[Protocol] = None + request_field: Optional[str] = None + response_field: Optional[str] = None + tool_calls_field: Optional[str] = None + headers: Dict[str, str] = Field(default_factory=dict) + auth_header_env: Optional[str] = None + + evaluators: Optional[List[EvaluatorOverride]] = None + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def _reject_legacy(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + legacy = _LEGACY_TOP_LEVEL_KEYS & set(data.keys()) + if legacy: + raise ValueError( + "agentops.yaml uses the new flat schema (see docs/concepts.md). " + f"Remove legacy keys: {sorted(legacy)}. The minimal config is " + "version + agent + dataset." + ) + return data + + @field_validator("version") + @classmethod + def _check_version(cls, value: int) -> int: + if value != 1: + raise ValueError( + f"agentops.yaml version must be 1 (got {value!r})" + ) + return value + + @field_validator("agent") + @classmethod + def _agent_non_empty(cls, value: str) -> str: + if not value.strip(): + raise ValueError("agent must be non-empty") + return value.strip() + + @model_validator(mode="after") + def _validate_protocol_compat(self) -> "AgentOpsConfig": + kind = classify_agent(self.agent, self.protocol).kind + if kind == "foundry_prompt" and self.protocol is not None: + raise ValueError( + "agent of the form 'name:version' is a Foundry prompt agent " + "and does not accept a 'protocol' field" + ) + if kind == "model_direct" and self.protocol is not None: + raise ValueError( + "agent of the form 'model:' does not accept a " + "'protocol' field" + ) + if kind != "http_json" and ( + self.request_field + or self.response_field + or self.tool_calls_field + or self.headers + or self.auth_header_env + ): + # Foundry hosted (responses/invocations) defines its own wire + # format. HTTP-only request/response shaping is invalid there. + if kind == "foundry_hosted" and self.protocol == "invocations": + # Invocations passes JSON through; users may need headers. + pass + else: + raise ValueError( + "request_field / response_field / tool_calls_field / " + "headers / auth_header_env are only valid for HTTP/JSON " + "or Foundry hosted (invocations) targets" + ) + return self + + def parsed_thresholds(self) -> List[Threshold]: + """Return the threshold dict parsed into structured rules.""" + return [ + Threshold.from_expression(metric, expression) + for metric, expression in self.thresholds.items() + ] + + def resolved_target(self) -> "TargetResolution": + """Return the resolved target classification.""" + return classify_agent(self.agent, self.protocol) + + +# --------------------------------------------------------------------------- +# Agent classifier +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class TargetResolution: + """Result of classifying the ``agent`` field.""" + + kind: TargetKind + protocol: Optional[Protocol] + raw: str + #: For ``foundry_prompt``: the agent name (left of the colon). + name: Optional[str] = None + #: For ``foundry_prompt``: the version (right of the colon). + version: Optional[str] = None + #: For ``foundry_hosted`` / ``http_json``: the target URL. + url: Optional[str] = None + #: For ``model_direct``: the deployment name. + deployment: Optional[str] = None + + +def _looks_like_foundry_url(url: str) -> bool: + """Return ``True`` when ``url`` matches a Foundry hosted endpoint pattern. + + Heuristic — Foundry URLs include the segment ``/agents/`` and the host + ends in a Foundry-recognized domain. We err on the side of accepting more + URLs as Foundry hosted (the user can force ``http-json`` via ``protocol``). + """ + lowered = url.lower() + foundry_domains = ( + ".azure.com", + ".azureml.ms", + ".cognitiveservices.azure.com", + ".services.ai.azure.com", + ".inference.ml.azure.com", + ".azurewebsites.net", # rare; users can override + ) + return any(domain in lowered for domain in foundry_domains) + + +def classify_agent( + agent: str, + protocol: Optional[Protocol] = None, +) -> TargetResolution: + """Classify the ``agent`` value into a target kind. + + Resolution table: + + +-------------------------+--------------------------+-----------------------+ + | ``agent`` value | ``protocol`` | ``TargetKind`` | + +=========================+==========================+=======================+ + | ``model:gpt-4o`` | n/a | ``model_direct`` | + +-------------------------+--------------------------+-----------------------+ + | ``my-rag:3`` | n/a | ``foundry_prompt`` | + +-------------------------+--------------------------+-----------------------+ + | ``https://...foundry`` | omitted or ``responses`` | ``foundry_hosted`` | + | (foundry-shaped URL) | | (responses) | + +-------------------------+--------------------------+-----------------------+ + | ``https://...foundry`` | ``invocations`` | ``foundry_hosted`` | + | | | (invocations) | + +-------------------------+--------------------------+-----------------------+ + | ``https://other-host`` | omitted or ``http-json`` | ``http_json`` | + +-------------------------+--------------------------+-----------------------+ + """ + raw = agent.strip() + + if raw.lower().startswith("model:"): + deployment = raw.split(":", 1)[1].strip() + if not deployment: + raise ValueError("model: prefix requires a deployment name") + return TargetResolution( + kind="model_direct", + protocol=None, + raw=raw, + deployment=deployment, + ) + + lowered = raw.lower() + if lowered.startswith(("http://", "https://")): + if _looks_like_foundry_url(raw): + resolved_protocol: Protocol = protocol or "responses" + if resolved_protocol not in {"responses", "invocations"}: + raise ValueError( + "Foundry hosted endpoints accept only protocol " + "'responses' or 'invocations'" + ) + return TargetResolution( + kind="foundry_hosted", + protocol=resolved_protocol, + raw=raw, + url=raw, + ) + + resolved_protocol = protocol or "http-json" + if resolved_protocol != "http-json": + raise ValueError( + "non-Foundry URLs must use protocol 'http-json' " + f"(got {resolved_protocol!r})" + ) + return TargetResolution( + kind="http_json", + protocol="http-json", + raw=raw, + url=raw, + ) + + if ":" in raw: + name, _, version = raw.partition(":") + name = name.strip() + version = version.strip() + if not name or not version: + raise ValueError( + "Foundry prompt agent must be 'name:version' " + f"(got {raw!r})" + ) + return TargetResolution( + kind="foundry_prompt", + protocol=None, + raw=raw, + name=name, + version=version, + ) + + raise ValueError( + f"unrecognized agent value {raw!r}: expected 'name:version', " + "'https://...', or 'model:'" + ) diff --git a/src/agentops/core/evaluators.py b/src/agentops/core/evaluators.py new file mode 100644 index 00000000..7024dc1e --- /dev/null +++ b/src/agentops/core/evaluators.py @@ -0,0 +1,405 @@ +"""Evaluator catalog and auto-selection for AgentOps 1.0. + +This module replaces the layered ``bundle.yaml`` system. There is no +user-facing ``scenario`` concept. Evaluators are picked from two inputs: + +1. The resolved target kind (agent vs model). Model targets only get the + baseline quality evaluators — agent-specific evaluators are skipped even + if the dataset contains those fields. +2. The shape of the dataset rows: + + * Always: baseline quality evaluators (Coherence, Fluency, Similarity, + F1Score). + * If rows include ``context``: add RAG evaluators (Groundedness, + Retrieval, Relevance, ResponseCompleteness). + * If rows include ``tool_calls`` or ``tool_definitions``: add agent + evaluators (TaskCompletion, ToolCallAccuracy, IntentResolution, + TaskAdherence, ToolSelection, ToolInputAccuracy). + +The :func:`select_evaluators` function returns a list of resolved +:class:`EvaluatorPreset` objects. Each preset carries its class name, the +input mapping it requires, the score key it produces, and a default +threshold. The runner uses these presets to instantiate +``azure-ai-evaluation`` evaluator classes against each dataset row. + +Power users can override the auto-selection by listing evaluator names in +``agentops.yaml`` under ``evaluators:``. When set, the override list is the +final word — no auto-detection runs. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, FrozenSet, List, Optional, Tuple + +from agentops.core.agentops_config import TargetKind, TargetResolution, Threshold + + +# --------------------------------------------------------------------------- +# Catalog +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EvaluatorPreset: + """Metadata for a single evaluator known to AgentOps. + + ``input_mapping`` keys are the parameter names the evaluator class + expects; values use the placeholder syntax ``$prompt``, ``$prediction``, + ``$context``, ``$expected``, ``$tool_calls``, ``$tool_definitions`` + which the runner resolves per row. + """ + + name: str + class_name: str + score_key: str + input_mapping: Dict[str, str] + default_threshold: Optional[Threshold] = None + #: Categories that this evaluator belongs to. Used by the inference rules. + categories: FrozenSet[str] = field(default_factory=frozenset) + #: Set when this evaluator is not safe to run for raw model deployments. + agent_only: bool = False + + +def _t(metric: str, criteria: str, value: float) -> Threshold: + return Threshold(metric=metric, criteria=criteria, value=value) # type: ignore[arg-type] + + +_QUALITY_BASELINE: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="CoherenceEvaluator", + class_name="CoherenceEvaluator", + score_key="coherence", + input_mapping={"query": "$prompt", "response": "$prediction"}, + default_threshold=_t("coherence", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="FluencyEvaluator", + class_name="FluencyEvaluator", + score_key="fluency", + input_mapping={"response": "$prediction"}, + default_threshold=_t("fluency", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="SimilarityEvaluator", + class_name="SimilarityEvaluator", + score_key="similarity", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("similarity", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="F1ScoreEvaluator", + class_name="F1ScoreEvaluator", + score_key="f1_score", + input_mapping={ + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("f1_score", ">=", 0.5), + categories=frozenset({"quality"}), + ), +) + + +_RAG_EVALUATORS: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="GroundednessEvaluator", + class_name="GroundednessEvaluator", + score_key="groundedness", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "context": "$context", + }, + default_threshold=_t("groundedness", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="RelevanceEvaluator", + class_name="RelevanceEvaluator", + score_key="relevance", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "context": "$context", + }, + default_threshold=_t("relevance", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="RetrievalEvaluator", + class_name="RetrievalEvaluator", + score_key="retrieval", + input_mapping={"query": "$prompt", "context": "$context"}, + default_threshold=_t("retrieval", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="ResponseCompletenessEvaluator", + class_name="ResponseCompletenessEvaluator", + score_key="response_completeness", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("response_completeness", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), +) + + +_TOOL_USE_EVALUATORS: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="TaskCompletionEvaluator", + class_name="TaskCompletionEvaluator", + score_key="task_completion", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "tool_calls": "$tool_calls", + "tool_definitions": "$tool_definitions", + }, + default_threshold=_t("task_completion", ">=", 3.0), + categories=frozenset({"agent"}), + agent_only=True, + ), + EvaluatorPreset( + name="ToolCallAccuracyEvaluator", + class_name="ToolCallAccuracyEvaluator", + score_key="tool_call_accuracy", + input_mapping={ + "query": "$prompt", + "tool_calls": "$tool_calls", + "tool_definitions": "$tool_definitions", + }, + default_threshold=_t("tool_call_accuracy", ">=", 0.7), + categories=frozenset({"agent"}), + agent_only=True, + ), + EvaluatorPreset( + name="IntentResolutionEvaluator", + class_name="IntentResolutionEvaluator", + score_key="intent_resolution", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "tool_definitions": "$tool_definitions", + }, + default_threshold=_t("intent_resolution", ">=", 3.0), + categories=frozenset({"agent"}), + agent_only=True, + ), + EvaluatorPreset( + name="TaskAdherenceEvaluator", + class_name="TaskAdherenceEvaluator", + score_key="task_adherence", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + }, + default_threshold=_t("task_adherence", ">=", 3.0), + categories=frozenset({"agent"}), + agent_only=True, + ), +) + + +_LATENCY = EvaluatorPreset( + name="avg_latency_seconds", + class_name="_BuiltinLatency", + score_key="avg_latency_seconds", + input_mapping={}, + default_threshold=_t("avg_latency_seconds", "<=", 10.0), + categories=frozenset({"runtime"}), +) + + +CATALOG: Dict[str, EvaluatorPreset] = { + preset.name: preset + for preset in ( + *_QUALITY_BASELINE, + *_RAG_EVALUATORS, + *_TOOL_USE_EVALUATORS, + _LATENCY, + ) +} + + +# --------------------------------------------------------------------------- +# Dataset shape detection +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DatasetShape: + """Boolean flags summarising the columns present in a dataset.""" + + has_context: bool + has_tool_calls: bool + has_tool_definitions: bool + row_count: int + + @property + def looks_rag(self) -> bool: + return self.has_context + + @property + def looks_tool_use(self) -> bool: + return self.has_tool_calls or self.has_tool_definitions + + +def detect_dataset_shape(dataset_path: Path, *, sample: int = 50) -> DatasetShape: + """Inspect up to ``sample`` rows of ``dataset_path`` and report the shape. + + Truthy values are required — empty strings, empty lists, and ``None`` do + not count as the field being present. + """ + if not dataset_path.exists(): + raise FileNotFoundError(f"dataset file not found: {dataset_path}") + + has_context = False + has_tool_calls = False + has_tool_definitions = False + count = 0 + + with dataset_path.open("r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if not stripped: + continue + count += 1 + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError( + f"{dataset_path}: invalid JSON on line {count}: {exc}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"{dataset_path}: line {count} is not a JSON object" + ) + + if not has_context and row.get("context"): + has_context = True + if not has_tool_calls and row.get("tool_calls"): + has_tool_calls = True + if not has_tool_definitions and row.get("tool_definitions"): + has_tool_definitions = True + + if count >= sample and ( + has_context and (has_tool_calls or has_tool_definitions) + ): + # Already saw both signals; no need to keep reading. + break + + if count == 0: + raise ValueError(f"{dataset_path}: dataset is empty") + + return DatasetShape( + has_context=has_context, + has_tool_calls=has_tool_calls, + has_tool_definitions=has_tool_definitions, + row_count=count, + ) + + +# --------------------------------------------------------------------------- +# Selection +# --------------------------------------------------------------------------- + + +def select_evaluators( + target: TargetResolution, + shape: DatasetShape, + *, + overrides: Optional[List[str]] = None, +) -> List[EvaluatorPreset]: + """Return the ordered list of evaluators to run. + + When ``overrides`` is provided it wins outright — the inference rules are + bypassed. Each name must exist in :data:`CATALOG` or a ``ValueError`` is + raised. + + Otherwise the rules are: + + * Always include the four baseline quality evaluators. + * If the target is a raw model, stop here. Agent-specific evaluators are + not meaningful (no tool calls, no retrieved context). + * If the dataset has ``context`` rows, add the RAG evaluators. + * If the dataset has ``tool_calls`` or ``tool_definitions``, add the agent + evaluators. + * Always append the runtime ``avg_latency_seconds`` evaluator. + """ + if overrides: + resolved: List[EvaluatorPreset] = [] + for name in overrides: + preset = CATALOG.get(name) + if preset is None: + known = ", ".join(sorted(CATALOG.keys())) + raise ValueError( + f"unknown evaluator override {name!r}. " + f"Known evaluators: {known}" + ) + resolved.append(preset) + return resolved + + selected: List[EvaluatorPreset] = list(_QUALITY_BASELINE) + + if _is_agent_target(target.kind): + if shape.looks_rag: + selected.extend(_RAG_EVALUATORS) + if shape.looks_tool_use: + selected.extend(_TOOL_USE_EVALUATORS) + + selected.append(_LATENCY) + return selected + + +def _is_agent_target(kind: TargetKind) -> bool: + return kind in {"foundry_prompt", "foundry_hosted", "http_json"} + + +def merge_thresholds( + presets: List[EvaluatorPreset], + user_thresholds: List[Threshold], +) -> List[Threshold]: + """Combine evaluator default thresholds with user overrides. + + User entries override the preset default for the same metric. Metrics + listed by the user that don't correspond to any selected preset are kept + as-is — the threshold engine will report them as unmet rather than + silently drop them. + """ + by_metric: Dict[str, Threshold] = {} + for preset in presets: + if preset.default_threshold is not None: + by_metric[preset.default_threshold.metric] = preset.default_threshold + for override in user_thresholds: + by_metric[override.metric] = override + # Preserve preset order, then append user-only metrics in original order. + ordered: List[Threshold] = [] + seen: set[str] = set() + for preset in presets: + if preset.default_threshold is not None: + metric = preset.default_threshold.metric + ordered.append(by_metric[metric]) + seen.add(metric) + for override in user_thresholds: + if override.metric not in seen: + ordered.append(override) + seen.add(override.metric) + return ordered diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py new file mode 100644 index 00000000..a411769c --- /dev/null +++ b/tests/unit/test_agentops_config.py @@ -0,0 +1,227 @@ +"""Tests for the flat ``agentops.yaml`` schema and agent classifier.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from agentops.core.agentops_config import ( + AgentOpsConfig, + Threshold, + classify_agent, +) + + +# --------------------------------------------------------------------------- +# classify_agent +# --------------------------------------------------------------------------- + + +class TestClassifyAgent: + def test_foundry_prompt_name_version(self) -> None: + result = classify_agent("my-rag:3") + assert result.kind == "foundry_prompt" + assert result.name == "my-rag" + assert result.version == "3" + assert result.protocol is None + + def test_foundry_prompt_rejects_empty_parts(self) -> None: + with pytest.raises(ValueError, match="name:version"): + classify_agent(":3") + with pytest.raises(ValueError, match="name:version"): + classify_agent("foo:") + + def test_model_direct(self) -> None: + result = classify_agent("model:gpt-4o-mini") + assert result.kind == "model_direct" + assert result.deployment == "gpt-4o-mini" + assert result.protocol is None + + def test_model_direct_rejects_empty_deployment(self) -> None: + with pytest.raises(ValueError, match="deployment name"): + classify_agent("model:") + + def test_foundry_hosted_default_protocol_responses(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + result = classify_agent(url) + assert result.kind == "foundry_hosted" + assert result.protocol == "responses" + assert result.url == url + + def test_foundry_hosted_invocations(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + result = classify_agent(url, protocol="invocations") + assert result.kind == "foundry_hosted" + assert result.protocol == "invocations" + + def test_foundry_hosted_rejects_http_json_protocol(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + with pytest.raises(ValueError, match="responses"): + classify_agent(url, protocol="http-json") + + def test_http_json_default_protocol(self) -> None: + url = "https://my-app.azurecontainerapps.io/chat" + result = classify_agent(url) + assert result.kind == "http_json" + assert result.protocol == "http-json" + + def test_http_json_rejects_responses_protocol(self) -> None: + url = "https://my-app.azurecontainerapps.io/chat" + with pytest.raises(ValueError, match="http-json"): + classify_agent(url, protocol="responses") + + def test_unrecognized_value(self) -> None: + with pytest.raises(ValueError, match="unrecognized"): + classify_agent("just-a-name") + + +# --------------------------------------------------------------------------- +# Threshold parser +# --------------------------------------------------------------------------- + + +class TestThresholdFromExpression: + @pytest.mark.parametrize( + "expression, expected_criteria, expected_value", + [ + (">=3", ">=", 3.0), + ("<=10", "<=", 10.0), + (">2.5", ">", 2.5), + ("<0.7", "<", 0.7), + ("==1", "==", 1.0), + (" >= 3 ", ">=", 3.0), + ], + ) + def test_comparison( + self, expression: str, expected_criteria: str, expected_value: float + ) -> None: + threshold = Threshold.from_expression("metric", expression) + assert threshold.criteria == expected_criteria + assert threshold.value == expected_value + + def test_bool_true(self) -> None: + threshold = Threshold.from_expression("metric", True) + assert threshold.criteria == "true" + assert threshold.value is None + + def test_bool_false_string(self) -> None: + threshold = Threshold.from_expression("metric", "false") + assert threshold.criteria == "false" + + def test_number_shorthand(self) -> None: + # bare number defaults to >= + threshold = Threshold.from_expression("metric", 3) + assert threshold.criteria == ">=" + assert threshold.value == 3.0 + + def test_invalid_expression(self) -> None: + with pytest.raises(ValueError, match="expected"): + Threshold.from_expression("metric", "approximately 3") + + def test_invalid_number(self) -> None: + with pytest.raises(ValueError, match="cannot parse"): + Threshold.from_expression("metric", ">=abc") + + +# --------------------------------------------------------------------------- +# AgentOpsConfig +# --------------------------------------------------------------------------- + + +class TestAgentOpsConfig: + def test_minimal_config(self, tmp_path) -> None: + cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") + assert cfg.version == 1 + assert cfg.agent == "my-rag:3" + assert cfg.thresholds == {} + + def test_resolved_target(self) -> None: + cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") + target = cfg.resolved_target() + assert target.kind == "foundry_prompt" + + def test_rejects_legacy_keys(self) -> None: + with pytest.raises(ValidationError) as exc_info: + AgentOpsConfig.model_validate( + { + "version": 1, + "agent": "my-rag:3", + "dataset": "./qa.jsonl", + "scenario": "rag", + } + ) + assert "legacy" in str(exc_info.value).lower() + + def test_rejects_extra_fields(self) -> None: + with pytest.raises(ValidationError): + AgentOpsConfig.model_validate( + { + "version": 1, + "agent": "my-rag:3", + "dataset": "./qa.jsonl", + "unknown_key": "x", + } + ) + + def test_rejects_wrong_version(self) -> None: + with pytest.raises(ValidationError, match="version must be 1"): + AgentOpsConfig(version=2, agent="my-rag:3", dataset="./qa.jsonl") + + def test_thresholds_parsed(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + thresholds={"groundedness": ">=3", "coherence": ">=3.5"}, + ) + parsed = {t.metric: t for t in cfg.parsed_thresholds()} + assert parsed["groundedness"].criteria == ">=" + assert parsed["groundedness"].value == 3.0 + assert parsed["coherence"].value == 3.5 + + def test_protocol_rejected_for_prompt_agent(self) -> None: + with pytest.raises(ValidationError, match="prompt agent"): + AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + protocol="responses", + ) + + def test_protocol_rejected_for_model_direct(self) -> None: + with pytest.raises(ValidationError, match="protocol"): + AgentOpsConfig( + version=1, + agent="model:gpt-4o", + dataset="./qa.jsonl", + protocol="http-json", + ) + + def test_http_fields_allowed_for_http_target(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="https://my-app.azurecontainerapps.io/chat", + dataset="./qa.jsonl", + request_field="message", + response_field="text", + ) + assert cfg.request_field == "message" + + def test_http_fields_rejected_for_prompt_agent(self) -> None: + with pytest.raises(ValidationError, match="HTTP/JSON"): + AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + request_field="message", + ) + + def test_evaluators_override(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + evaluators=[{"name": "GroundednessEvaluator"}], # type: ignore[list-item] + ) + assert cfg.evaluators is not None + assert cfg.evaluators[0].name == "GroundednessEvaluator" diff --git a/tests/unit/test_evaluators.py b/tests/unit/test_evaluators.py new file mode 100644 index 00000000..8e6b031c --- /dev/null +++ b/tests/unit/test_evaluators.py @@ -0,0 +1,226 @@ +"""Tests for the evaluator catalog and auto-selection rules.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agentops.core.agentops_config import Threshold, classify_agent +from agentops.core.evaluators import ( + CATALOG, + DatasetShape, + detect_dataset_shape, + merge_thresholds, + select_evaluators, +) + + +# --------------------------------------------------------------------------- +# detect_dataset_shape +# --------------------------------------------------------------------------- + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + import json + + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") + + +class TestDetectDatasetShape: + def test_quality_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "qa.jsonl" + _write_jsonl( + path, + [ + {"input": "hello", "expected": "hi"}, + {"input": "bye", "expected": "goodbye"}, + ], + ) + shape = detect_dataset_shape(path) + assert shape.row_count == 2 + assert not shape.looks_rag + assert not shape.looks_tool_use + + def test_rag_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "rag.jsonl" + _write_jsonl( + path, + [ + {"input": "q", "expected": "a", "context": "Paris is the capital."}, + ], + ) + shape = detect_dataset_shape(path) + assert shape.looks_rag + + def test_tool_use_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "tools.jsonl" + _write_jsonl( + path, + [ + { + "input": "weather?", + "expected": "sunny", + "tool_calls": [{"name": "get_weather", "args": {}}], + }, + ], + ) + shape = detect_dataset_shape(path) + assert shape.looks_tool_use + + def test_empty_context_does_not_count(self, tmp_path: Path) -> None: + path = tmp_path / "empty_ctx.jsonl" + _write_jsonl( + path, + [{"input": "q", "expected": "a", "context": ""}], + ) + shape = detect_dataset_shape(path) + assert not shape.looks_rag + + def test_empty_dataset_raises(self, tmp_path: Path) -> None: + path = tmp_path / "empty.jsonl" + path.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + detect_dataset_shape(path) + + def test_invalid_json_raises(self, tmp_path: Path) -> None: + path = tmp_path / "bad.jsonl" + path.write_text("not json\n", encoding="utf-8") + with pytest.raises(ValueError, match="invalid JSON"): + detect_dataset_shape(path) + + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + detect_dataset_shape(tmp_path / "missing.jsonl") + + +# --------------------------------------------------------------------------- +# select_evaluators +# --------------------------------------------------------------------------- + + +_PROMPT_AGENT = classify_agent("my-rag:3") +_MODEL_DIRECT = classify_agent("model:gpt-4o") +_HTTP_AGENT = classify_agent("https://my-app.azurecontainerapps.io/chat") + + +def _shape(*, context: bool = False, tool_calls: bool = False, tool_defs: bool = False) -> DatasetShape: + return DatasetShape( + has_context=context, + has_tool_calls=tool_calls, + has_tool_definitions=tool_defs, + row_count=10, + ) + + +class TestSelectEvaluators: + def test_quality_baseline_always_present(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape()) + names = [p.name for p in result] + assert "CoherenceEvaluator" in names + assert "FluencyEvaluator" in names + assert "SimilarityEvaluator" in names + assert "F1ScoreEvaluator" in names + assert "avg_latency_seconds" in names + + def test_quality_only_for_quality_dataset(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape()) + names = [p.name for p in result] + assert "GroundednessEvaluator" not in names + assert "TaskCompletionEvaluator" not in names + + def test_rag_evaluators_added_with_context(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(context=True)) + names = [p.name for p in result] + for evaluator in [ + "GroundednessEvaluator", + "RelevanceEvaluator", + "RetrievalEvaluator", + "ResponseCompletenessEvaluator", + ]: + assert evaluator in names + + def test_tool_use_added_with_tool_calls(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(tool_calls=True)) + names = [p.name for p in result] + assert "TaskCompletionEvaluator" in names + assert "ToolCallAccuracyEvaluator" in names + + def test_tool_use_added_with_tool_definitions(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(tool_defs=True)) + names = [p.name for p in result] + assert "TaskCompletionEvaluator" in names + + def test_combined_rag_and_tools(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(context=True, tool_calls=True)) + names = [p.name for p in result] + assert "GroundednessEvaluator" in names + assert "TaskCompletionEvaluator" in names + + def test_model_direct_skips_agent_evaluators(self) -> None: + # Even if the dataset has context/tool_calls, model targets stay quality-only. + result = select_evaluators( + _MODEL_DIRECT, _shape(context=True, tool_calls=True) + ) + names = [p.name for p in result] + assert "GroundednessEvaluator" not in names + assert "TaskCompletionEvaluator" not in names + assert "CoherenceEvaluator" in names + + def test_http_agent_treated_like_agent(self) -> None: + result = select_evaluators(_HTTP_AGENT, _shape(context=True)) + names = [p.name for p in result] + assert "GroundednessEvaluator" in names + + def test_overrides_bypass_inference(self) -> None: + result = select_evaluators( + _PROMPT_AGENT, + _shape(context=True, tool_calls=True), + overrides=["CoherenceEvaluator"], + ) + names = [p.name for p in result] + assert names == ["CoherenceEvaluator"] + + def test_unknown_override_raises(self) -> None: + with pytest.raises(ValueError, match="unknown evaluator"): + select_evaluators(_PROMPT_AGENT, _shape(), overrides=["NotAnEvaluator"]) + + +# --------------------------------------------------------------------------- +# merge_thresholds +# --------------------------------------------------------------------------- + + +class TestMergeThresholds: + def test_user_override_wins(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + user = [Threshold(metric="coherence", criteria=">=", value=4.0)] + merged = merge_thresholds(presets, user) + coherence = [t for t in merged if t.metric == "coherence"][0] + assert coherence.value == 4.0 + + def test_preset_default_used_when_no_override(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + merged = merge_thresholds(presets, user_thresholds=[]) + # CoherenceEvaluator default is >=3.0 + coherence = [t for t in merged if t.metric == "coherence"][0] + assert coherence.value == 3.0 + + def test_user_only_metric_appended(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + user = [Threshold(metric="custom_metric", criteria=">=", value=1.0)] + merged = merge_thresholds(presets, user) + names = [t.metric for t in merged] + assert "custom_metric" in names + + +# --------------------------------------------------------------------------- +# CATALOG +# --------------------------------------------------------------------------- + + +def test_catalog_keys_match_preset_names() -> None: + for name, preset in CATALOG.items(): + assert preset.name == name From f0d11c7cfecfbb8b6b196558218d5aa2c0691a3d Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 27 Apr 2026 11:10:28 -0300 Subject: [PATCH 02/70] feat(pipeline): add 1.0 evaluation orchestrator with 4 backends Implements the new pipeline namespace introduced in the revamp: - core/results.py: RunResult/RowResult/ThresholdEvaluation/ComparisonInfo Pydantic models for the stable results.json contract. - pipeline/invocations.py: 4 target backends (model_direct, foundry_prompt, foundry_hosted Responses+Invocations, http_json) behind a single invoke() dispatcher. - pipeline/runtime.py: lazy azure-ai-evaluation evaluator instantiation, per-row execution, score extraction; baked-in handling for AI-assisted vs safety classes. - pipeline/thresholds.py: criteria evaluation against aggregated metrics with 'missing' fallback. - pipeline/comparison.py + reporter.py: --baseline support and report.md rendering with delta tables. - pipeline/orchestrator.py: end-to-end run_evaluation() that classifies the agent, infers evaluators from dataset shape, runs invocations, applies thresholds, and writes results.json + report.md. - core/config_loader.py: new load_agentops_config() loader for agentops.yaml. - tests/integration/test_pipeline_smoke.py: 2 end-to-end tests against an in-process HTTP echo server (no Azure dependencies); covers baseline comparison. Refs #107 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/core/config_loader.py | 6 + src/agentops/core/results.py | 116 ++++++++ src/agentops/pipeline/__init__.py | 8 + src/agentops/pipeline/comparison.py | 108 +++++++ src/agentops/pipeline/invocations.py | 356 +++++++++++++++++++++++ src/agentops/pipeline/orchestrator.py | 297 +++++++++++++++++++ src/agentops/pipeline/reporter.py | 130 +++++++++ src/agentops/pipeline/runtime.py | 222 ++++++++++++++ src/agentops/pipeline/thresholds.py | 84 ++++++ tests/integration/test_pipeline_smoke.py | 139 +++++++++ 10 files changed, 1466 insertions(+) create mode 100644 src/agentops/core/results.py create mode 100644 src/agentops/pipeline/__init__.py create mode 100644 src/agentops/pipeline/comparison.py create mode 100644 src/agentops/pipeline/invocations.py create mode 100644 src/agentops/pipeline/orchestrator.py create mode 100644 src/agentops/pipeline/reporter.py create mode 100644 src/agentops/pipeline/runtime.py create mode 100644 src/agentops/pipeline/thresholds.py create mode 100644 tests/integration/test_pipeline_smoke.py diff --git a/src/agentops/core/config_loader.py b/src/agentops/core/config_loader.py index 7a22abd3..1827c049 100644 --- a/src/agentops/core/config_loader.py +++ b/src/agentops/core/config_loader.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ValidationError +from agentops.core.agentops_config import AgentOpsConfig from agentops.core.models import ( BundleConfig, BundleRef, @@ -31,6 +32,11 @@ def _load_model(path: Path, model_cls: Type[TModel], label: str) -> TModel: raise ValueError(f"{label} validation error: {exc}") from exc +def load_agentops_config(path: Path) -> AgentOpsConfig: + """Load the flat 1.0 ``agentops.yaml`` schema.""" + return _load_model(path, AgentOpsConfig, "AgentOpsConfig") + + def load_workspace_config(path: Path) -> WorkspaceConfig: return _load_model(path, WorkspaceConfig, "WorkspaceConfig") diff --git a/src/agentops/core/results.py b/src/agentops/core/results.py new file mode 100644 index 00000000..80a91d55 --- /dev/null +++ b/src/agentops/core/results.py @@ -0,0 +1,116 @@ +"""Result dataclasses for the AgentOps 1.0 pipeline. + +These shapes are written to ``results.json`` after every ``agentops eval`` run +and consumed by the reporter and comparison logic. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class RowMetric(BaseModel): + """A single evaluator score for one dataset row.""" + + name: str + value: Optional[float] = None + error: Optional[str] = None + + +class RowResult(BaseModel): + """One evaluated dataset row.""" + + row_index: int + input: str + expected: Optional[str] = None + response: str = "" + context: Optional[str] = None + latency_seconds: Optional[float] = None + tool_calls: Optional[List[Any]] = None + metrics: List[RowMetric] = Field(default_factory=list) + error: Optional[str] = None + + +class ThresholdEvaluation(BaseModel): + """A pass/fail check for a single metric on the run aggregate.""" + + metric: str + criteria: str + expected: str + actual: str + passed: bool + + +class RunSummary(BaseModel): + """Top-level pass/fail summary of an evaluation run.""" + + items_total: int + items_passed_all: int + items_pass_rate: float + thresholds_total: int + thresholds_passed: int + threshold_pass_rate: float + overall_passed: bool + + +class TargetInfo(BaseModel): + """Resolved target information (echoed into results.json).""" + + kind: str + raw: str + protocol: Optional[str] = None + name: Optional[str] = None + version: Optional[str] = None + url: Optional[str] = None + deployment: Optional[str] = None + + +class ComparisonMetric(BaseModel): + """Per-metric delta between the current run and a baseline.""" + + metric: str + current: Optional[float] = None + baseline: Optional[float] = None + delta: Optional[float] = None + direction: str # "improved" | "regressed" | "unchanged" + + +class ComparisonRow(BaseModel): + """Per-row regression / improvement against a baseline.""" + + row_index: int + current_passed: bool + baseline_passed: Optional[bool] = None + direction: str # "improved" | "regressed" | "unchanged" | "new" + + +class ComparisonInfo(BaseModel): + """Comparison block included when ``--baseline`` was provided.""" + + baseline_path: str + baseline_started_at: Optional[str] = None + baseline_overall_passed: Optional[bool] = None + metrics: List[ComparisonMetric] = Field(default_factory=list) + rows: List[ComparisonRow] = Field(default_factory=list) + + +class RunResult(BaseModel): + """Full ``results.json`` payload.""" + + version: int = 1 + started_at: str + finished_at: str + duration_seconds: float + target: TargetInfo + dataset_path: str + evaluators: List[str] = Field(default_factory=list) + rows: List[RowResult] = Field(default_factory=list) + aggregate_metrics: Dict[str, float] = Field(default_factory=dict) + thresholds: List[ThresholdEvaluation] = Field(default_factory=list) + summary: RunSummary + comparison: Optional[ComparisonInfo] = None + config: Dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") diff --git a/src/agentops/pipeline/__init__.py b/src/agentops/pipeline/__init__.py new file mode 100644 index 00000000..c53271ca --- /dev/null +++ b/src/agentops/pipeline/__init__.py @@ -0,0 +1,8 @@ +"""AgentOps 1.0 evaluation pipeline. + +Public entry point: :func:`agentops.pipeline.orchestrator.run_evaluation`. +""" + +from agentops.pipeline.orchestrator import run_evaluation + +__all__ = ["run_evaluation"] diff --git a/src/agentops/pipeline/comparison.py b/src/agentops/pipeline/comparison.py new file mode 100644 index 00000000..dd57d35a --- /dev/null +++ b/src/agentops/pipeline/comparison.py @@ -0,0 +1,108 @@ +"""``--baseline`` comparison helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Dict, List, Optional + +from agentops.core.results import ( + ComparisonInfo, + ComparisonMetric, + ComparisonRow, + RunResult, +) + + +def load_baseline(path: Path) -> RunResult: + """Load a previous ``results.json`` for comparison.""" + if not path.exists(): + raise FileNotFoundError(f"baseline file not found: {path}") + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + return RunResult.model_validate(payload) + + +def _direction(current: Optional[float], baseline: Optional[float]) -> str: + if current is None or baseline is None: + return "unchanged" + if current > baseline: + return "improved" + if current < baseline: + return "regressed" + return "unchanged" + + +def _row_passed(row_metrics: List[Dict[str, float | None]]) -> bool: + """Best-effort proxy: a row is "passing" when no metric reports an error.""" + return all("error" not in metric or not metric["error"] for metric in row_metrics) + + +def build_comparison( + *, + current: RunResult, + baseline: RunResult, + baseline_path: Path, +) -> ComparisonInfo: + metrics: List[ComparisonMetric] = [] + metric_names = sorted(set(current.aggregate_metrics) | set(baseline.aggregate_metrics)) + for name in metric_names: + current_value = current.aggregate_metrics.get(name) + baseline_value = baseline.aggregate_metrics.get(name) + delta = ( + current_value - baseline_value + if current_value is not None and baseline_value is not None + else None + ) + metrics.append( + ComparisonMetric( + metric=name, + current=current_value, + baseline=baseline_value, + delta=delta, + direction=_direction(current_value, baseline_value), + ) + ) + + rows: List[ComparisonRow] = [] + baseline_by_index = {row.row_index: row for row in baseline.rows} + for row in current.rows: + baseline_row = baseline_by_index.get(row.row_index) + current_pass = row.error is None and all( + m.value is not None or m.error is None for m in row.metrics + ) + if baseline_row is None: + rows.append( + ComparisonRow( + row_index=row.row_index, + current_passed=current_pass, + baseline_passed=None, + direction="new", + ) + ) + continue + baseline_pass = baseline_row.error is None and all( + m.value is not None or m.error is None for m in baseline_row.metrics + ) + if current_pass and not baseline_pass: + direction = "improved" + elif baseline_pass and not current_pass: + direction = "regressed" + else: + direction = "unchanged" + rows.append( + ComparisonRow( + row_index=row.row_index, + current_passed=current_pass, + baseline_passed=baseline_pass, + direction=direction, + ) + ) + + return ComparisonInfo( + baseline_path=str(baseline_path), + baseline_started_at=baseline.started_at, + baseline_overall_passed=baseline.summary.overall_passed, + metrics=metrics, + rows=rows, + ) diff --git a/src/agentops/pipeline/invocations.py b/src/agentops/pipeline/invocations.py new file mode 100644 index 00000000..da7ac978 --- /dev/null +++ b/src/agentops/pipeline/invocations.py @@ -0,0 +1,356 @@ +"""Target invocation backends for AgentOps 1.0. + +Each backend is a single function with the signature:: + + invoke( + target: TargetResolution, + config: AgentOpsConfig, + row: dict[str, Any], + *, + timeout: float, + ) -> InvocationResult + +The orchestrator dispatches based on :attr:`TargetResolution.kind`. All Azure +SDK imports are lazy so the package imports without optional dependencies. +""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from agentops.core.agentops_config import AgentOpsConfig, TargetResolution + + +@dataclass +class InvocationResult: + """Outcome of invoking the target on one dataset row.""" + + response: str + latency_seconds: float + tool_calls: Optional[List[Any]] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _credential() -> Any: + from azure.identity import DefaultAzureCredential # noqa: WPS433 + + return DefaultAzureCredential(exclude_developer_cli_credential=True) + + +def _get_token(scope: str) -> str: + return _credential().get_token(scope).token + + +def _project_endpoint_from_env() -> str: + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise RuntimeError( + "Missing AZURE_AI_FOUNDRY_PROJECT_ENDPOINT environment variable. " + "Foundry targets require a project endpoint URL." + ) + return endpoint.rstrip("/") + + +def _row_input(row: Dict[str, Any]) -> str: + value = row.get("input") + if value is None: + raise ValueError("dataset row is missing required 'input' field") + return str(value) + + +def _http_request_json( + *, + method: str, + url: str, + headers: Dict[str, str], + body: Optional[Dict[str, Any]] = None, + timeout: float, +) -> Dict[str, Any]: + encoded = json.dumps(body or {}).encode("utf-8") if method != "GET" else None + request = urllib.request.Request( + url=url, data=encoded, method=method, headers=headers + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + payload = response.read().decode("utf-8") + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise RuntimeError( + f"HTTP {exc.code} from {url}: {detail or exc.reason}" + ) from exc + if not payload: + return {} + return json.loads(payload) + + +def _dot_path(payload: Any, path: str) -> Any: + """Resolve ``a.b.c`` or ``a.0.b`` against a JSON-like object.""" + current = payload + for token in path.split("."): + if current is None: + return None + if isinstance(current, list): + try: + current = current[int(token)] + except (ValueError, IndexError): + return None + continue + if isinstance(current, dict): + current = current.get(token) + continue + return None + return current + + +def _extract_responses_text(payload: Dict[str, Any]) -> str: + direct = payload.get("output_text") + if isinstance(direct, str) and direct.strip(): + return direct.strip() + + output = payload.get("output") + if isinstance(output, list): + parts: List[str] = [] + for item in output: + if not isinstance(item, dict): + continue + if ( + item.get("type") in {"message", "assistant_message"} + or item.get("role") == "assistant" + ): + content = item.get("content") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for chunk in content: + if isinstance(chunk, dict): + text = chunk.get("text") or chunk.get("output_text") + if isinstance(text, str): + parts.append(text) + elif isinstance(chunk, str): + parts.append(chunk) + if parts: + return "\n".join(parts).strip() + + raise ValueError("Foundry response did not include assistant output text") + + +def _extract_responses_tool_calls(payload: Dict[str, Any]) -> Optional[List[Any]]: + output = payload.get("output") + if not isinstance(output, list): + return None + calls: List[Any] = [] + for item in output: + if isinstance(item, dict) and item.get("type") in { + "tool_call", + "function_call", + }: + calls.append(item) + return calls or None + + +# --------------------------------------------------------------------------- +# Backends +# --------------------------------------------------------------------------- + + +def _invoke_model_direct( + target: TargetResolution, + config: AgentOpsConfig, # noqa: ARG001 + row: Dict[str, Any], + *, + timeout: float, # noqa: ARG001 +) -> InvocationResult: + from azure.ai.projects import AIProjectClient # noqa: WPS433 + + project_endpoint = _project_endpoint_from_env() + client = AIProjectClient(endpoint=project_endpoint, credential=_credential()) + openai_client = client.get_openai_client() + + assert target.deployment is not None + started = time.perf_counter() + response = openai_client.chat.completions.create( + model=target.deployment, + messages=[{"role": "user", "content": _row_input(row)}], + ) + elapsed = time.perf_counter() - started + + text = "" + if response.choices: + message = response.choices[0].message + if message and message.content: + text = message.content.strip() + if not text: + raise RuntimeError("model_direct invocation returned empty content") + + return InvocationResult(response=text, latency_seconds=elapsed) + + +def _invoke_foundry_prompt( + target: TargetResolution, + config: AgentOpsConfig, # noqa: ARG001 + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + project_endpoint = _project_endpoint_from_env() + token = _get_token("https://ai.azure.com/.default") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + } + + assert target.name is not None and target.version is not None + body = { + "input": [{"role": "user", "content": _row_input(row)}], + "agent_reference": { + "type": "agent_reference", + "name": target.name, + "version": target.version, + }, + } + + started = time.perf_counter() + payload = _http_request_json( + method="POST", + url=f"{project_endpoint}/openai/v1/responses", + headers=headers, + body=body, + timeout=timeout, + ) + elapsed = time.perf_counter() - started + + text = _extract_responses_text(payload) + tool_calls = _extract_responses_tool_calls(payload) + return InvocationResult( + response=text, latency_seconds=elapsed, tool_calls=tool_calls, + ) + + +def _invoke_foundry_hosted( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + assert target.url is not None + token = _get_token("https://ai.azure.com/.default") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + **config.headers, + } + + if target.protocol == "responses": + body = {"input": [{"role": "user", "content": _row_input(row)}]} + url = target.url.rstrip("/") + if not url.endswith("/responses"): + url = f"{url}/responses" + + started = time.perf_counter() + payload = _http_request_json( + method="POST", url=url, headers=headers, body=body, timeout=timeout + ) + elapsed = time.perf_counter() - started + + return InvocationResult( + response=_extract_responses_text(payload), + latency_seconds=elapsed, + tool_calls=_extract_responses_tool_calls(payload), + ) + + return _invoke_http_json(target, config, row, timeout=timeout) + + +def _invoke_http_json( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + assert target.url is not None + headers: Dict[str, str] = {"Content-Type": "application/json", **config.headers} + if config.auth_header_env: + token = os.getenv(config.auth_header_env) + if not token: + raise RuntimeError( + f"auth_header_env {config.auth_header_env!r} is set in config but " + "the environment variable is empty" + ) + headers["Authorization"] = f"Bearer {token}" + + request_field = config.request_field or "message" + body: Dict[str, Any] = {request_field: _row_input(row)} + + started = time.perf_counter() + payload = _http_request_json( + method="POST", + url=target.url, + headers=headers, + body=body, + timeout=timeout, + ) + elapsed = time.perf_counter() - started + + response_path = config.response_field or "text" + response_text = _dot_path(payload, response_path) + if response_text is None: + for fallback in ("response", "output", "content", "message", "text"): + response_text = payload.get(fallback) + if response_text: + break + if response_text is None: + raise ValueError( + f"HTTP/JSON response did not contain field {response_path!r}; " + f"got top-level keys: {sorted(payload.keys())}" + ) + if not isinstance(response_text, str): + response_text = json.dumps(response_text, ensure_ascii=False) + + tool_calls: Optional[List[Any]] = None + if config.tool_calls_field: + extracted = _dot_path(payload, config.tool_calls_field) + if isinstance(extracted, list): + tool_calls = extracted + + return InvocationResult( + response=response_text.strip(), + latency_seconds=elapsed, + tool_calls=tool_calls, + ) + + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + + +def invoke( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + """Dispatch to the right backend based on the resolved target kind.""" + if target.kind == "model_direct": + return _invoke_model_direct(target, config, row, timeout=timeout) + if target.kind == "foundry_prompt": + return _invoke_foundry_prompt(target, config, row, timeout=timeout) + if target.kind == "foundry_hosted": + return _invoke_foundry_hosted(target, config, row, timeout=timeout) + if target.kind == "http_json": + return _invoke_http_json(target, config, row, timeout=timeout) + raise ValueError(f"unknown target kind: {target.kind}") diff --git a/src/agentops/pipeline/orchestrator.py b/src/agentops/pipeline/orchestrator.py new file mode 100644 index 00000000..ad5eecd8 --- /dev/null +++ b/src/agentops/pipeline/orchestrator.py @@ -0,0 +1,297 @@ +"""End-to-end evaluation orchestrator for AgentOps 1.0. + +This is the single entry point exercised by ``agentops eval``. It loads the +flat config, classifies the target, infers evaluators from the dataset shape, +invokes the target row-by-row, runs each evaluator, applies thresholds, and +writes ``results.json`` and ``report.md``. +""" + +from __future__ import annotations + +import json +import logging +import statistics +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from agentops.core.agentops_config import AgentOpsConfig, Threshold, classify_agent +from agentops.core.evaluators import ( + detect_dataset_shape, + merge_thresholds, + select_evaluators, +) +from agentops.core.results import ( + RowMetric, + RowResult, + RunResult, + RunSummary, + TargetInfo, +) +from agentops.pipeline import comparison as comparison_module +from agentops.pipeline import invocations, reporter, runtime, thresholds + +logger = logging.getLogger("agentops.pipeline") + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +@dataclass +class RunOptions: + config_path: Path + output_dir: Path + baseline_path: Optional[Path] = None + timeout_seconds: float = 120.0 + dataset_override: Optional[Path] = None + agent_override: Optional[str] = None + + +def run_evaluation( + config: AgentOpsConfig, + *, + options: RunOptions, +) -> RunResult: + """Run a full evaluation and persist artifacts. Returns the RunResult.""" + started_at = datetime.now(timezone.utc) + started_perf = time.perf_counter() + + target = classify_agent( + options.agent_override or config.agent, + config.protocol, + ) + + dataset_path = options.dataset_override or _resolve_dataset_path(config, options) + shape = detect_dataset_shape(dataset_path) + + overrides = ( + [override.name for override in config.evaluators] if config.evaluators else None + ) + presets = select_evaluators(target, shape, overrides=overrides) + user_thresholds = [ + Threshold.from_expression(metric, expr) + for metric, expr in config.thresholds.items() + ] + threshold_rules = merge_thresholds(presets, user_thresholds) + + evaluator_runtimes = runtime.load_evaluators(presets) + + rows: List[RowResult] = [] + for index, row in enumerate(_iter_dataset(dataset_path)): + rows.append( + _evaluate_row( + row=row, + index=index, + target=target, + config=config, + evaluators=evaluator_runtimes, + timeout=options.timeout_seconds, + ) + ) + + aggregate = _aggregate_metrics(rows) + threshold_results = thresholds.evaluate(threshold_rules, aggregate) + summary = _summarize(rows, threshold_results) + + finished_at = datetime.now(timezone.utc) + duration = time.perf_counter() - started_perf + + result = RunResult( + started_at=started_at.isoformat(), + finished_at=finished_at.isoformat(), + duration_seconds=duration, + target=TargetInfo( + kind=target.kind, + raw=target.raw, + protocol=target.protocol, + name=target.name, + version=target.version, + url=target.url, + deployment=target.deployment, + ), + dataset_path=str(dataset_path), + evaluators=[preset.name for preset in presets], + rows=rows, + aggregate_metrics=aggregate, + thresholds=threshold_results, + summary=summary, + config={ + "version": config.version, + "agent": config.agent, + "thresholds": dict(config.thresholds), + }, + ) + + if options.baseline_path is not None: + baseline = comparison_module.load_baseline(options.baseline_path) + result.comparison = comparison_module.build_comparison( + current=result, + baseline=baseline, + baseline_path=options.baseline_path, + ) + + _persist(result, options.output_dir) + return result + + +def exit_code_from(result: RunResult) -> int: + """Translate a run's outcome into the ``agentops`` CLI contract. + + * ``0`` — success, all thresholds passed. + * ``2`` — invocations succeeded but a threshold failed. + * ``1`` — runtime errors are raised as exceptions before this is called. + """ + return 0 if result.summary.overall_passed else 2 + + +# --------------------------------------------------------------------------- +# Dataset +# --------------------------------------------------------------------------- + + +def _resolve_dataset_path(config: AgentOpsConfig, options: RunOptions) -> Path: + candidate = config.dataset + if candidate.is_absolute() and candidate.exists(): + return candidate + base = options.config_path.parent + resolved = (base / candidate).resolve() + if not resolved.exists(): + raise FileNotFoundError(f"dataset not found: {resolved}") + return resolved + + +def _iter_dataset(path: Path) -> Iterable[Dict[str, Any]]: + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + stripped = line.strip() + if not stripped: + continue + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError( + f"{path}: invalid JSON on line {line_number}: {exc}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"{path}: line {line_number} is not a JSON object" + ) + yield row + + +# --------------------------------------------------------------------------- +# Per-row execution +# --------------------------------------------------------------------------- + + +def _evaluate_row( + *, + row: Dict[str, Any], + index: int, + target, + config: AgentOpsConfig, + evaluators: List[runtime.EvaluatorRuntime], + timeout: float, +) -> RowResult: + try: + invocation = invocations.invoke(target, config, row, timeout=timeout) + except Exception as exc: # noqa: BLE001 + logger.warning("row %d invocation failed: %s", index, exc) + return RowResult( + row_index=index, + input=str(row.get("input", "")), + expected=row.get("expected"), + response="", + context=row.get("context"), + error=str(exc), + ) + + metrics: List[RowMetric] = [] + for evaluator in evaluators: + metric = runtime.run_evaluator( + evaluator, + row=row, + response=invocation.response, + latency_seconds=invocation.latency_seconds, + ) + metrics.append(metric) + + return RowResult( + row_index=index, + input=str(row.get("input", "")), + expected=row.get("expected"), + response=invocation.response, + context=row.get("context"), + latency_seconds=invocation.latency_seconds, + tool_calls=invocation.tool_calls, + metrics=metrics, + ) + + +# --------------------------------------------------------------------------- +# Aggregation +# --------------------------------------------------------------------------- + + +def _aggregate_metrics(rows: List[RowResult]) -> Dict[str, float]: + by_metric: Dict[str, List[float]] = {} + for row in rows: + for metric in row.metrics: + if metric.value is None: + continue + by_metric.setdefault(metric.name, []).append(metric.value) + aggregate: Dict[str, float] = {} + for name, values in by_metric.items(): + if values: + aggregate[name] = statistics.fmean(values) + return aggregate + + +def _summarize( + rows: List[RowResult], + threshold_results, +) -> RunSummary: + items_total = len(rows) + items_passed_all = sum( + 1 + for row in rows + if row.error is None and all(m.error is None for m in row.metrics) + ) + items_pass_rate = items_passed_all / items_total if items_total else 0.0 + thresholds_total = len(threshold_results) + thresholds_passed = sum(1 for t in threshold_results if t.passed) + threshold_pass_rate = ( + thresholds_passed / thresholds_total if thresholds_total else 1.0 + ) + overall = items_total > 0 and threshold_pass_rate == 1.0 and items_passed_all > 0 + return RunSummary( + items_total=items_total, + items_passed_all=items_passed_all, + items_pass_rate=items_pass_rate, + thresholds_total=thresholds_total, + thresholds_passed=thresholds_passed, + threshold_pass_rate=threshold_pass_rate, + overall_passed=overall, + ) + + +# --------------------------------------------------------------------------- +# Persistence +# --------------------------------------------------------------------------- + + +def _persist(result: RunResult, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + results_path = output_dir / "results.json" + report_path = output_dir / "report.md" + + payload = result.model_dump(mode="json") + results_path.write_text( + json.dumps(payload, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + report_path.write_text(reporter.render(result), encoding="utf-8") diff --git a/src/agentops/pipeline/reporter.py b/src/agentops/pipeline/reporter.py new file mode 100644 index 00000000..c6a6bff4 --- /dev/null +++ b/src/agentops/pipeline/reporter.py @@ -0,0 +1,130 @@ +"""Reporter for AgentOps 1.0 — generates ``report.md`` from a ``RunResult``.""" + +from __future__ import annotations + +from typing import List, Optional + +from agentops.core.results import ( + ComparisonInfo, + ComparisonMetric, + ComparisonRow, + RowResult, + RunResult, + ThresholdEvaluation, +) + + +def render(result: RunResult) -> str: + """Render a RunResult into a Markdown report.""" + lines: List[str] = [] + lines.append("# AgentOps Evaluation Report") + lines.append("") + overall = "✅ PASS" if result.summary.overall_passed else "❌ FAIL" + lines.append(f"**Result:** {overall}") + lines.append(f"- **Target:** `{result.target.raw}` ({result.target.kind})") + if result.target.protocol: + lines.append(f"- **Protocol:** {result.target.protocol}") + lines.append(f"- **Dataset:** `{result.dataset_path}`") + lines.append(f"- **Started:** {result.started_at}") + lines.append(f"- **Duration:** {result.duration_seconds:.2f}s") + lines.append(f"- **Rows:** {result.summary.items_total}") + lines.append("") + + if result.aggregate_metrics: + lines.append("## Metrics") + lines.append("") + lines.append("| Metric | Value |") + lines.append("| --- | --- |") + for name, value in sorted(result.aggregate_metrics.items()): + lines.append(f"| {name} | {value:.3f} |") + lines.append("") + + if result.thresholds: + lines.append("## Thresholds") + lines.append("") + lines.append("| Metric | Expected | Actual | Status |") + lines.append("| --- | --- | --- | --- |") + for threshold in result.thresholds: + lines.append(_threshold_row(threshold)) + lines.append("") + + if result.comparison is not None: + lines.extend(_render_comparison(result.comparison)) + lines.append("") + + error_rows = [row for row in result.rows if row.error] + if error_rows: + lines.append("## Failed Invocations") + lines.append("") + lines.append("| Row | Error |") + lines.append("| --- | --- |") + for row in error_rows: + lines.append(f"| {row.row_index} | {_short(row.error or '', 200)} |") + lines.append("") + + lines.append("## Rows") + lines.append("") + lines.append("| # | Latency (s) | Metrics |") + lines.append("| --- | --- | --- |") + for row in result.rows: + lines.append(_row_summary(row)) + lines.append("") + return "\n".join(lines) + + +def _threshold_row(threshold: ThresholdEvaluation) -> str: + status = "✅" if threshold.passed else "❌" + return f"| {threshold.metric} | `{threshold.expected}` | `{threshold.actual}` | {status} |" + + +def _row_summary(row: RowResult) -> str: + parts = [] + for metric in row.metrics: + if metric.error: + parts.append(f"{metric.name}=ERR") + elif metric.value is not None: + parts.append(f"{metric.name}={metric.value:.2f}") + metrics_str = ", ".join(parts) if parts else "—" + latency = f"{row.latency_seconds:.2f}" if row.latency_seconds is not None else "—" + return f"| {row.row_index} | {latency} | {metrics_str} |" + + +def _short(text: str, limit: int) -> str: + text = text.replace("\n", " ").replace("|", "\\|") + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _render_comparison(comparison: ComparisonInfo) -> List[str]: + lines = ["## Comparison vs Baseline", ""] + lines.append(f"**Baseline:** `{comparison.baseline_path}`") + if comparison.baseline_started_at: + lines.append(f"**Baseline run:** {comparison.baseline_started_at}") + lines.append("") + + lines.append("| Metric | Baseline | Current | Δ | Direction |") + lines.append("| --- | --- | --- | --- | --- |") + for metric in comparison.metrics: + lines.append(_comparison_metric_row(metric)) + lines.append("") + + regressed = [r for r in comparison.rows if r.direction == "regressed"] + improved = [r for r in comparison.rows if r.direction == "improved"] + if regressed or improved: + lines.append("**Per-row changes:**") + if regressed: + lines.append( + "- ❌ Regressed rows: " + ", ".join(str(r.row_index) for r in regressed) + ) + if improved: + lines.append( + "- ✅ Improved rows: " + ", ".join(str(r.row_index) for r in improved) + ) + return lines + + +def _comparison_metric_row(metric: ComparisonMetric) -> str: + arrow = {"improved": "🟢", "regressed": "🔴", "unchanged": "⚪"}[metric.direction] + baseline = f"{metric.baseline:.3f}" if metric.baseline is not None else "—" + current = f"{metric.current:.3f}" if metric.current is not None else "—" + delta = f"{metric.delta:+.3f}" if metric.delta is not None else "—" + return f"| {metric.metric} | {baseline} | {current} | {delta} | {arrow} {metric.direction} |" diff --git a/src/agentops/pipeline/runtime.py b/src/agentops/pipeline/runtime.py new file mode 100644 index 00000000..19a05767 --- /dev/null +++ b/src/agentops/pipeline/runtime.py @@ -0,0 +1,222 @@ +"""Evaluator runtime for AgentOps 1.0. + +Each :class:`EvaluatorPreset` from the catalog is instantiated lazily from +``azure.ai.evaluation`` and run against one dataset row. The runtime hides +SDK details (``model_config`` for AI-assisted evaluators, ``azure_ai_project`` +for safety evaluators, kwarg mapping, score extraction). +""" + +from __future__ import annotations + +import importlib +import inspect +import os +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agentops.core.evaluators import EvaluatorPreset +from agentops.core.results import RowMetric + +# Evaluator classes that require an evaluator model via ``model_config``. +_AI_ASSISTED = { + "GroundednessEvaluator", + "RelevanceEvaluator", + "CoherenceEvaluator", + "FluencyEvaluator", + "SimilarityEvaluator", + "RetrievalEvaluator", + "ResponseCompletenessEvaluator", + "QAEvaluator", + "IntentResolutionEvaluator", + "TaskAdherenceEvaluator", + "ToolCallAccuracyEvaluator", + "TaskCompletionEvaluator", + "ToolSelectionEvaluator", + "ToolInputAccuracyEvaluator", +} + +# Evaluator classes that require ``azure_ai_project``. +_SAFETY = { + "ViolenceEvaluator", + "SexualEvaluator", + "SelfHarmEvaluator", + "HateUnfairnessEvaluator", + "ContentSafetyEvaluator", + "ProtectedMaterialEvaluator", +} + + +@dataclass +class EvaluatorRuntime: + """A loaded, ready-to-call evaluator.""" + + preset: EvaluatorPreset + callable: Any # evaluator instance or sentinel for "latency" + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + + +def _credential() -> Any: + from azure.identity import DefaultAzureCredential # noqa: WPS433 + + return DefaultAzureCredential(exclude_developer_cli_credential=True) + + +def _model_config() -> Dict[str, str]: + endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") + deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") or os.getenv( + "AZURE_AI_MODEL_DEPLOYMENT_NAME" + ) + api_version = os.getenv("AZURE_OPENAI_API_VERSION") + + missing = [] + if not endpoint: + missing.append("AZURE_OPENAI_ENDPOINT") + if not deployment: + missing.append("AZURE_OPENAI_DEPLOYMENT") + if missing: + raise RuntimeError( + "AI-assisted evaluators require an evaluator model. " + "Missing environment variables: " + ", ".join(missing) + ) + + config: Dict[str, str] = { + "azure_endpoint": endpoint, # type: ignore[dict-item] + "azure_deployment": deployment, # type: ignore[dict-item] + } + if api_version: + config["api_version"] = api_version + return config + + +def _project_endpoint() -> str: + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise RuntimeError( + "Safety evaluators require AZURE_AI_FOUNDRY_PROJECT_ENDPOINT." + ) + return endpoint + + +_LATENCY_SENTINEL = object() + + +def load_evaluator(preset: EvaluatorPreset) -> EvaluatorRuntime: + """Instantiate one evaluator. Raises a clear error if the SDK is missing.""" + if preset.class_name == "_latency": + return EvaluatorRuntime(preset=preset, callable=_LATENCY_SENTINEL) + + try: + module = importlib.import_module("azure.ai.evaluation") + except ImportError as exc: + raise RuntimeError( + "Evaluators require the 'azure-ai-evaluation' package. " + "Install with: pip install azure-ai-evaluation" + ) from exc + + cls = getattr(module, preset.class_name, None) + if cls is None: + raise RuntimeError( + f"Evaluator class {preset.class_name!r} not found in azure.ai.evaluation" + ) + + init_kwargs: Dict[str, Any] = {} + if preset.class_name in _AI_ASSISTED: + init_kwargs["model_config"] = _model_config() + if preset.class_name in _SAFETY: + init_kwargs["azure_ai_project"] = _project_endpoint() + init_kwargs["credential"] = _credential() + + try: + instance = cls(**init_kwargs) if inspect.isclass(cls) else cls + except TypeError: + # Some evaluators reject unexpected kwargs (e.g. F1ScoreEvaluator). + instance = cls() if inspect.isclass(cls) else cls + + return EvaluatorRuntime(preset=preset, callable=instance) + + +def load_evaluators(presets: List[EvaluatorPreset]) -> List[EvaluatorRuntime]: + return [load_evaluator(preset) for preset in presets] + + +# --------------------------------------------------------------------------- +# Execution +# --------------------------------------------------------------------------- + + +_PLACEHOLDERS = { + "$prompt": "input", + "$prediction": "response", + "$expected": "expected", + "$context": "context", + "$tool_calls": "tool_calls", + "$tool_definitions": "tool_definitions", +} + + +def _resolve_kwargs( + mapping: Dict[str, str], + *, + row: Dict[str, Any], + response: str, +) -> Dict[str, Any]: + resolved: Dict[str, Any] = {} + merged = {**row, "response": response, "input": row.get("input")} + for kwarg, placeholder in mapping.items(): + if not isinstance(placeholder, str) or not placeholder.startswith("$"): + resolved[kwarg] = placeholder + continue + source_key = _PLACEHOLDERS.get(placeholder) + if source_key is None: + raise ValueError(f"unknown evaluator placeholder {placeholder!r}") + value = merged.get(source_key) + if value is None: + continue + resolved[kwarg] = value + return resolved + + +def _extract_score(payload: Any, score_key: str) -> Optional[float]: + if payload is None: + return None + if isinstance(payload, (int, float)): + return float(payload) + if not isinstance(payload, dict): + return None + for candidate in ( + score_key, + f"{score_key}_score", + f"gpt_{score_key}", + "score", + ): + value = payload.get(candidate) + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + return None + + +def run_evaluator( + runtime: EvaluatorRuntime, + *, + row: Dict[str, Any], + response: str, + latency_seconds: float, +) -> RowMetric: + """Execute one evaluator on one row. Captures errors so the run continues.""" + preset = runtime.preset + if runtime.callable is _LATENCY_SENTINEL: + return RowMetric(name=preset.score_key, value=float(latency_seconds)) + + try: + kwargs = _resolve_kwargs(preset.input_mapping, row=row, response=response) + result = runtime.callable(**kwargs) + score = _extract_score(result, preset.score_key) + return RowMetric(name=preset.score_key, value=score) + except Exception as exc: # noqa: BLE001 + return RowMetric(name=preset.score_key, error=str(exc)) diff --git a/src/agentops/pipeline/thresholds.py b/src/agentops/pipeline/thresholds.py new file mode 100644 index 00000000..801016e0 --- /dev/null +++ b/src/agentops/pipeline/thresholds.py @@ -0,0 +1,84 @@ +"""Threshold evaluation against parsed :class:`Threshold` rules.""" + +from __future__ import annotations + +from typing import Dict, List + +from agentops.core.agentops_config import Threshold +from agentops.core.results import ThresholdEvaluation + + +def evaluate( + rules: List[Threshold], + metrics: Dict[str, float], +) -> List[ThresholdEvaluation]: + """Apply each rule against the aggregate metric value. + + Missing metrics produce a failed evaluation with ``actual="missing"`` so + the report can show the gap clearly rather than crashing the run. + """ + results: List[ThresholdEvaluation] = [] + for rule in rules: + actual_value = metrics.get(rule.metric) + + if rule.criteria in {"true", "false"}: + expected = rule.criteria + actual = "missing" + passed = False + if actual_value is not None: + actual_bool = actual_value == 1.0 + actual = "true" if actual_bool else "false" + passed = actual == expected + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected, + actual=actual, + passed=passed, + ) + ) + continue + + if rule.value is None: + raise ValueError( + f"threshold for {rule.metric!r} requires a numeric value" + ) + + target = float(rule.value) + expected_str = f"{rule.criteria}{target:g}" + if actual_value is None: + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected_str, + actual="missing", + passed=False, + ) + ) + continue + + if rule.criteria == ">=": + passed = actual_value >= target + elif rule.criteria == ">": + passed = actual_value > target + elif rule.criteria == "<=": + passed = actual_value <= target + elif rule.criteria == "<": + passed = actual_value < target + elif rule.criteria == "==": + passed = actual_value == target + else: + raise ValueError(f"unsupported criteria {rule.criteria!r}") + + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected_str, + actual=f"{actual_value:g}", + passed=passed, + ) + ) + return results diff --git a/tests/integration/test_pipeline_smoke.py b/tests/integration/test_pipeline_smoke.py new file mode 100644 index 00000000..8c203c7f --- /dev/null +++ b/tests/integration/test_pipeline_smoke.py @@ -0,0 +1,139 @@ +"""End-to-end smoke test for the AgentOps 1.0 pipeline. + +Spins up a tiny HTTP server, points an ``agentops.yaml`` at it, runs the +orchestrator without any Azure dependencies (no AI-assisted evaluators), and +asserts the resulting ``results.json`` and ``report.md``. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +from agentops.core.agentops_config import AgentOpsConfig +from agentops.core.config_loader import load_agentops_config +from agentops.pipeline.orchestrator import ( + RunOptions, + exit_code_from, + run_evaluation, +) + + +class _EchoHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8")) + message = body.get("message", "") + payload = json.dumps({"text": f"echo: {message}"}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args, **kwargs) -> None: # noqa: D401 + pass + + +@pytest.fixture() +def echo_server(): + server = HTTPServer(("127.0.0.1", 0), _EchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}/chat" + finally: + server.shutdown() + thread.join(timeout=1) + + +def _write_dataset(path: Path) -> None: + rows = [ + {"input": "say hi", "expected": "hi"}, + {"input": "say bye", "expected": "bye"}, + ] + path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + +def _write_config(path: Path, *, agent_url: str, dataset: Path) -> None: + payload = { + "version": 1, + "agent": agent_url, + "dataset": str(dataset), + "evaluators": [{"name": "F1ScoreEvaluator"}], # avoids Azure model dependency + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_http_pipeline_end_to_end(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + + config_path = tmp_path / "agentops.yaml" + _write_config(config_path, agent_url=echo_server, dataset=dataset) + + config = load_agentops_config(config_path) + assert isinstance(config, AgentOpsConfig) + assert config.agent == echo_server + + output_dir = tmp_path / "results" + options = RunOptions( + config_path=config_path, + output_dir=output_dir, + timeout_seconds=10.0, + ) + + result = run_evaluation(config, options=options) + + assert (output_dir / "results.json").exists() + assert (output_dir / "report.md").exists() + assert result.summary.items_total == 2 + assert result.target.kind == "http_json" + assert "f1_score" in result.aggregate_metrics + assert result.rows[0].response.startswith("echo:") + + payload = json.loads((output_dir / "results.json").read_text(encoding="utf-8")) + assert payload["version"] == 1 + assert payload["target"]["url"] == echo_server + + code = exit_code_from(result) + assert code in (0, 2) + + +def test_http_pipeline_with_baseline(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config_path = tmp_path / "agentops.yaml" + _write_config(config_path, agent_url=echo_server, dataset=dataset) + config = load_agentops_config(config_path) + + baseline_dir = tmp_path / "baseline" + run_evaluation( + config, + options=RunOptions( + config_path=config_path, + output_dir=baseline_dir, + timeout_seconds=10.0, + ), + ) + + current_dir = tmp_path / "current" + result = run_evaluation( + config, + options=RunOptions( + config_path=config_path, + output_dir=current_dir, + baseline_path=baseline_dir / "results.json", + timeout_seconds=10.0, + ), + ) + + assert result.comparison is not None + assert any(metric.metric == "f1_score" for metric in result.comparison.metrics) + report_text = (current_dir / "report.md").read_text(encoding="utf-8") + assert "Comparison vs Baseline" in report_text From a1c6a31d4d0b04b72d23a8a391272d0d319408ac Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 27 Apr 2026 11:18:46 -0300 Subject: [PATCH 03/70] feat(cli,init,docs): wire flat schema into CLI and docs - cli/app.py: 'agentops eval run' auto-detects flat agentops.yaml and routes through the new pipeline; legacy run.yaml continues to work. New --baseline option enabled on the flat path. - cli/app.py: 'agentops init --flat' bootstraps the 1.0 layout (agentops.yaml + .agentops/data/smoke.jsonl). - services/initializer.py: new initialize_flat_workspace() helper with seed templates. - templates/agentops.yaml + smoke.jsonl: 1.0 starter assets, packaged via pyproject. - templates/workflows/agentops-eval.yml: defaults to agentops.yaml then falls back to .agentops/run.yaml. - README.md: adds a 1.0 Quickstart section before the legacy quickstart. - docs/tutorial-1.0-quickstart.md: end-to-end walkthrough for the flat schema. - examples/flat-quickstart/: a runnable example folder. - CHANGELOG.md: Unreleased section documenting the revamp. - tests: 4 new tests (CLI flat path + init_flat + baseline) on top of the existing 322. Full suite at 326 green (test_browse failure is pre-existing and unrelated). Refs #107 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 25 +++ README.md | 63 ++++++- docs/tutorial-1.0-quickstart.md | 113 +++++++++++++ examples/flat-quickstart/README.md | 15 ++ examples/flat-quickstart/agentops.yaml | 23 +++ examples/flat-quickstart/dataset.jsonl | 3 + pyproject.toml | 2 + src/agentops/cli/app.py | 155 +++++++++++++++++- src/agentops/services/initializer.py | 45 +++++ src/agentops/templates/agentops.yaml | 26 +++ src/agentops/templates/smoke.jsonl | 3 + .../templates/workflows/agentops-eval.yml | 13 +- tests/integration/test_cli_flat_schema.py | 119 ++++++++++++++ tests/unit/test_init_flat.py | 42 +++++ 14 files changed, 634 insertions(+), 13 deletions(-) create mode 100644 docs/tutorial-1.0-quickstart.md create mode 100644 examples/flat-quickstart/README.md create mode 100644 examples/flat-quickstart/agentops.yaml create mode 100644 examples/flat-quickstart/dataset.jsonl create mode 100644 src/agentops/templates/agentops.yaml create mode 100644 src/agentops/templates/smoke.jsonl create mode 100644 tests/integration/test_cli_flat_schema.py create mode 100644 tests/unit/test_init_flat.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b0bddb9..5c871d4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,31 @@ All notable changes to this project will be documented in this file. This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres to [Semantic Versioning](https://semver.org/). +# Changelog + +All notable changes to this project will be documented in this file. +This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] — 1.0 revamp + +### Added +- **Flat `agentops.yaml` schema** — a three-line minimal config (`version`, `agent`, `dataset`) that replaces the multi-file `run.yaml` + `bundle.yaml` + `dataset.yaml` for the common case. Implemented in `src/agentops/core/agentops_config.py`. +- **Automatic target classification** — the `agent:` value is classified as `foundry_prompt` (`name:version`), `foundry_hosted` (`https://...services.ai.azure.com/...`), `http_json` (any other URL), or `model_direct` (`model:`). +- **Automatic evaluator inference** — evaluators are picked from the target type and the dataset shape (presence of `context`, `tool_calls`, `tool_definitions`). The `scenario` concept is gone from the user-facing surface. Implemented in `src/agentops/core/evaluators.py`. +- **New `pipeline/` namespace** — `orchestrator.py`, `invocations.py`, `runtime.py`, `thresholds.py`, `comparison.py`, `reporter.py`. Single end-to-end pipeline that supports all four target kinds, including Foundry hosted endpoints (Responses + Invocations protocols). +- **`agentops eval run --baseline `** — compare a run against any previous `results.json` and append a `Comparison vs Baseline` table to the report. +- **`agentops init --flat`** — bootstrap a 1.0 workspace (root `agentops.yaml` + seed dataset). The legacy multi-file workspace remains available as the default for backward compatibility. +- **Tutorial: 1.0 quickstart** — `docs/tutorial-1.0-quickstart.md`. + +### Changed +- **`agentops eval run`** auto-detects the schema and routes flat configs through the new pipeline; legacy `run.yaml` files continue to work unchanged. +- **GitHub Actions workflow template** now defaults to `agentops.yaml` (falls back to `.agentops/run.yaml`). +- **README** gains a 1.0 Quickstart section before the legacy quickstart. + +### Engineering +- 326 unit + integration tests passing (was 319 before the revamp). New end-to-end smoke covers the HTTP backend without requiring Azure credentials. +- Foundry SDK calls remain lazy and never pass `api_version` to `get_openai_client()`. + ## [0.1.7] - 2026-04-21 ### Added diff --git a/README.md b/README.md index 379421bb..0b3bba96 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,68 @@ Exit code contract: - `2` execution succeeded but one or more thresholds failed - `1` runtime or configuration error -## Quickstart +## Quickstart (1.0) + +> The 1.0 release introduces a flat, three-line config for the most common case. The legacy multi-file workspace is still supported — see [`docs/how-it-works.md`](docs/how-it-works.md) for details. + +### 1) Install + +```bash +python -m venv .venv +python -m pip install -U pip +python -m pip install agentops-toolkit +``` + +### 2) Bootstrap + +```bash +agentops init --flat +``` + +This writes a single `agentops.yaml` at the project root and a tiny seed dataset at `.agentops/data/smoke.jsonl`. Edit `agentops.yaml` to point at your agent. + +### 3) Configure your agent + +Pick one of these forms for the `agent:` field — AgentOps classifies the target automatically: + +```yaml +agent: "my-rag:3" # Foundry prompt agent (name:version) +agent: "https://...services.ai.azure.com/.../agents/" # Foundry hosted endpoint +agent: "https://api.example.com/chat" # any HTTP/JSON agent (ACA, AKS, custom) +agent: "model:gpt-4o" # raw Foundry model deployment +``` + +Evaluators are inferred from the dataset shape (rows with `context` → RAG evaluators, rows with `tool_calls`/`tool_definitions` → agent-workflow evaluators). The full minimal config is: + +```yaml +version: 1 +agent: "my-rag:3" +dataset: .agentops/data/smoke.jsonl +``` + +### 4) Run + +```bash +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +agentops eval run +``` + +Outputs land in `.agentops/results/latest/`: + +- `results.json` — machine-readable (versioned, stable schema) +- `report.md` — human-readable, PR-friendly + +To compare against a previous run, pass `--baseline`: + +```bash +agentops eval run --baseline .agentops/results/baseline/results.json +``` + +The report grows a `Comparison vs Baseline` section with per-metric deltas. + +--- + +## Quickstart (legacy multi-file layout)

Quickstart demo: agentops init and eval run diff --git a/docs/tutorial-1.0-quickstart.md b/docs/tutorial-1.0-quickstart.md new file mode 100644 index 00000000..7dfbcedb --- /dev/null +++ b/docs/tutorial-1.0-quickstart.md @@ -0,0 +1,113 @@ +# Tutorial: 1.0 minimal quickstart + +This tutorial covers the simplest end-to-end AgentOps 1.0 flow: bootstrap a workspace, point it at any agent, and run an evaluation. + +## What you will build + +- A flat `agentops.yaml` at your project root. +- A small JSONL dataset. +- One `agentops eval run` execution producing `results.json` and `report.md`. + +The rest of the toolkit (legacy bundles, multi-file workspaces, custom adapters) still works, but is not required for the common case. + +## Prerequisites + +- Python 3.11 or later. +- Access to a target agent or model. Choose one: + - A **Foundry prompt agent** identified by `name:version` (for example `customer-support:3`). + - A **Foundry hosted endpoint** (`https://*.services.ai.azure.com/.../agents/`). + - A **generic HTTP/JSON agent** deployed anywhere (ACA, AKS, your own server). + - A **raw Foundry model deployment** (e.g. `gpt-4o`). +- For Foundry targets: `az login` (or a service principal) and `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` set. +- For AI-assisted evaluators (Coherence, Groundedness, etc.): `AZURE_OPENAI_ENDPOINT` and `AZURE_OPENAI_DEPLOYMENT` set. + +## 1. Install + +```bash +python -m venv .venv +python -m pip install -U pip +python -m pip install agentops-toolkit +``` + +## 2. Bootstrap the project + +```bash +agentops init --flat +``` + +This creates two files: + +- `agentops.yaml` — your evaluation config (3 lines + comments). +- `.agentops/data/smoke.jsonl` — a 3-row seed dataset. + +## 3. Configure your agent + +Open `agentops.yaml` and set the `agent:` field. The classifier infers the target kind from the value: + +| Value | Resolves to | +| -------------------------------------------------------- | ------------------------------------ | +| `"customer-support:3"` | Foundry prompt agent (`name:version`) | +| `"https://.services.ai.azure.com/.../agents/"` | Foundry hosted endpoint | +| `"https://api.example.com/chat"` | Generic HTTP/JSON agent | +| `"model:gpt-4o"` | Raw Foundry model deployment | + +The full minimal config is just: + +```yaml +version: 1 +agent: "customer-support:3" +dataset: .agentops/data/smoke.jsonl +``` + +## 4. Run the evaluation + +Set credentials and run: + +```bash +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +agentops eval run +``` + +Outputs: + +``` +.agentops/results/latest/ +├── results.json +└── report.md +``` + +The CLI prints `Threshold status: PASSED` (exit code `0`) or `FAILED` (exit code `2`) so you can wire it into CI directly. + +## 5. Compare against a baseline + +Save a known-good run and use it as a baseline: + +```bash +agentops eval run --output .agentops/results/baseline +# ... change your prompt, model, or dataset ... +agentops eval run --baseline .agentops/results/baseline/results.json +``` + +`report.md` now includes a `Comparison vs Baseline` section with per-metric deltas (🟢 improved / 🔴 regressed / ⚪ unchanged). + +## Where evaluators come from + +You did not pick evaluators — AgentOps inferred them: + +- **Always:** Coherence, Fluency, Similarity, F1Score, average latency. +- **If your dataset rows include `context`:** Groundedness, Relevance, Retrieval, ResponseCompleteness. +- **If your dataset rows include `tool_calls` or `tool_definitions`:** TaskCompletion, ToolCallAccuracy, IntentResolution, TaskAdherence. + +To override the auto-selection, list evaluator class names in `agentops.yaml`: + +```yaml +evaluators: + - GroundednessEvaluator + - CoherenceEvaluator +``` + +## Where to go next + +- [`docs/how-it-works.md`](how-it-works.md) — architecture and request flow. +- [`docs/ci-github-actions.md`](ci-github-actions.md) — wire AgentOps into PR checks with OIDC auth. +- The existing tutorials still apply if you stay on the legacy multi-file layout. diff --git a/examples/flat-quickstart/README.md b/examples/flat-quickstart/README.md new file mode 100644 index 00000000..f949eefa --- /dev/null +++ b/examples/flat-quickstart/README.md @@ -0,0 +1,15 @@ +# Flat quickstart example + +The smallest possible AgentOps 1.0 setup: an `agentops.yaml`, a JSONL dataset, and one CLI command. + +## Run + +```bash +cd examples/flat-quickstart +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +agentops eval run --config agentops.yaml --output ./out +``` + +Edit `agentops.yaml` first to point `agent:` at one of your real targets (Foundry prompt agent, Foundry hosted endpoint, generic HTTP/JSON agent, or `model:`). + +Outputs land in `./out/results.json` and `./out/report.md`. diff --git a/examples/flat-quickstart/agentops.yaml b/examples/flat-quickstart/agentops.yaml new file mode 100644 index 00000000..8efda340 --- /dev/null +++ b/examples/flat-quickstart/agentops.yaml @@ -0,0 +1,23 @@ +version: 1 + +# Pick one of these and remove the others: +# +# Foundry prompt agent (name:version): +agent: "my-rag:1" +# +# Foundry hosted endpoint: +# agent: "https://.services.ai.azure.com/api/projects//agents/" +# +# Generic HTTP/JSON agent (ACA, AKS, custom server): +# agent: "https://api.example.com/chat" +# +# Raw Foundry model deployment: +# agent: "model:gpt-4o" + +dataset: ./dataset.jsonl + +# Optional thresholds (override the auto-selected defaults): +# thresholds: +# coherence: ">=3" +# groundedness: ">=3" +# avg_latency_seconds: "<=10" diff --git a/examples/flat-quickstart/dataset.jsonl b/examples/flat-quickstart/dataset.jsonl new file mode 100644 index 00000000..5f37d6a1 --- /dev/null +++ b/examples/flat-quickstart/dataset.jsonl @@ -0,0 +1,3 @@ +{"input": "What is Microsoft Foundry?", "expected": "Foundry is Microsoft's enterprise AI platform for building, evaluating, and deploying agents."} +{"input": "What does AgentOps do?", "expected": "AgentOps standardizes evaluation workflows for Foundry agents and models."} +{"input": "How do I run an evaluation?", "expected": "Run `agentops eval run` after configuring agentops.yaml."} diff --git a/pyproject.toml b/pyproject.toml index d3cad410..b463da45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ where = ["src"] [tool.setuptools.package-data] "agentops.templates" = [ + "agentops.yaml", + "smoke.jsonl", "config.yaml", "run.yaml", "run-rag.yaml", diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 478536c7..15d21bbe 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -151,8 +151,39 @@ def cmd_init( "--path", help="Workspace directory to initialise.", ), + flat: bool = typer.Option( + False, + "--flat", + help="Use the 1.0 minimal layout (agentops.yaml at project root + .agentops/data/smoke.jsonl).", + ), ) -> None: - """Initialise an AgentOps workspace (creates .agentops/).""" + """Initialise an AgentOps workspace. + + The default layout creates a full ``.agentops/`` workspace (legacy). Pass + ``--flat`` to bootstrap the simpler 1.0 layout — a single + ``agentops.yaml`` at the project root and a tiny seed dataset. + """ + if flat: + from agentops.services.initializer import initialize_flat_workspace + + log.debug("cmd_init flat=True force=%s dir=%s", force, directory) + try: + result = initialize_flat_workspace(directory=directory, force=force) + except Exception as exc: + typer.echo(f"Error: failed to initialize workspace: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo("Initialized AgentOps 1.0 workspace.") + for created in result.created_files: + typer.echo(f" + created {created}") + for overwritten in result.overwritten_files: + typer.echo(f" ~ overwritten {overwritten}") + for skipped in result.skipped_files: + typer.echo(f" - skipped {skipped}") + typer.echo("") + typer.echo("Edit agentops.yaml to point at your agent, then run: agentops eval run") + return + from agentops.services.initializer import initialize_workspace log.debug("cmd_init called force=%s dir=%s", force, directory) @@ -194,33 +225,61 @@ def cmd_eval_run( typer.Option( "--config", "-c", - help="Path to run.yaml (default: .agentops/run.yaml).", + help="Path to agentops.yaml (1.0) or run.yaml (legacy). " + "Defaults: agentops.yaml, then .agentops/run.yaml.", ), ] = None, output: Annotated[ Path | None, typer.Option("--output", "-o", help="Output directory for results."), ] = None, + baseline: Annotated[ + Path | None, + typer.Option( + "--baseline", + help="Path to a previous results.json to compare this run against (1.0 only).", + ), + ] = None, report_format: Annotated[ str, typer.Option("--format", "-f", help="Report format: md, html, or all.") ] = "md", ) -> None: - """Run an evaluation defined in a run.yaml file.""" - from agentops.services.runner import run_evaluation - + """Run an evaluation defined in agentops.yaml (1.0) or run.yaml (legacy).""" if report_format not in ("md", "html", "all"): typer.echo("Error: --format must be md, html, or all.", err=True) raise typer.Exit(code=1) + config_path = _resolve_eval_config_path(config) log.debug( - "cmd_eval_run called config=%s output=%s format=%s", - config, + "cmd_eval_run called config=%s output=%s format=%s baseline=%s", + config_path, output, report_format, + baseline, ) + + if _is_flat_schema(config_path): + _run_flat_schema_eval( + config_path=config_path, + output=output, + baseline=baseline, + ) + return + + if baseline is not None: + typer.echo( + "Error: --baseline is only supported with the 1.0 agentops.yaml schema.", + err=True, + ) + raise typer.Exit(code=1) + + from agentops.services.runner import run_evaluation as legacy_run_evaluation + try: - run_result = run_evaluation( - config_path=config, output_override=output, report_format=report_format + run_result = legacy_run_evaluation( + config_path=config_path, + output_override=output, + report_format=report_format, ) except Exception as exc: typer.echo(f"Error: evaluation failed: {exc}", err=True) @@ -237,6 +296,84 @@ def cmd_eval_run( typer.echo("Threshold status: PASSED") +def _resolve_eval_config_path(config: Path | None) -> Path: + if config is not None: + return config + flat = Path("agentops.yaml") + if flat.exists(): + return flat + return Path(".agentops/run.yaml") + + +def _is_flat_schema(config_path: Path) -> bool: + """Return True when config_path is a 1.0 agentops.yaml file.""" + if not config_path.exists(): + return False + try: + from agentops.utils.yaml import load_yaml + except ImportError: + return False + try: + data = load_yaml(config_path) + except Exception: + return False + if not isinstance(data, dict): + return False + # Flat schema markers: 'agent' (string) at top-level, no legacy 'target'/'bundle'. + if "target" in data or "bundle" in data: + return False + return isinstance(data.get("agent"), str) + + +def _run_flat_schema_eval( + *, + config_path: Path, + output: Path | None, + baseline: Path | None, +) -> None: + from agentops.core.config_loader import load_agentops_config + from agentops.pipeline.orchestrator import ( + RunOptions, + exit_code_from, + run_evaluation, + ) + + try: + config_obj = load_agentops_config(config_path) + except Exception as exc: + typer.echo(f"Error: failed to load {config_path}: {exc}", err=True) + raise typer.Exit(code=1) from exc + + output_dir = output or _default_flat_output_dir(config_path) + + options = RunOptions( + config_path=config_path.resolve(), + output_dir=output_dir, + baseline_path=baseline.resolve() if baseline else None, + ) + + try: + result = run_evaluation(config_obj, options=options) + except Exception as exc: + typer.echo(f"Error: evaluation failed: {exc}", err=True) + raise typer.Exit(code=1) from exc + + typer.echo(f"Evaluation output directory: {output_dir}") + typer.echo(f"results.json: {output_dir / 'results.json'}") + typer.echo(f"report.md: {output_dir / 'report.md'}") + if result.summary.overall_passed: + typer.echo("Threshold status: PASSED") + return + typer.echo("Threshold status: FAILED") + raise typer.Exit(code=exit_code_from(result)) + + +def _default_flat_output_dir(config_path: Path) -> Path: + base = config_path.parent / ".agentops" / "results" + return base / "latest" + + + @eval_app.command("compare") def cmd_eval_compare( runs: Annotated[ diff --git a/src/agentops/services/initializer.py b/src/agentops/services/initializer.py index 5f6e3559..92b43e2b 100644 --- a/src/agentops/services/initializer.py +++ b/src/agentops/services/initializer.py @@ -95,3 +95,48 @@ def initialize_workspace(directory: Path, force: bool = False) -> InitResult: result.created_files.append(file_path) return result + + +# --------------------------------------------------------------------------- +# 1.0 flat workspace (agentops.yaml at project root + minimal seed dataset) +# --------------------------------------------------------------------------- + + +_FLAT_FILES: Dict[str, str] = { + "agentops.yaml": "agentops.yaml", + ".agentops/data/smoke.jsonl": "smoke.jsonl", +} + + +def initialize_flat_workspace(directory: Path, force: bool = False) -> InitResult: + """Bootstrap the AgentOps 1.0 workspace. + + Creates ``agentops.yaml`` at the project root and a tiny seed dataset at + ``.agentops/data/smoke.jsonl``. This is the recommended starting point for + new projects; the legacy multi-file workspace remains available via + :func:`initialize_workspace`. + """ + project_root = directory.resolve() + result = InitResult(workspace_dir=project_root / ".agentops") + + templates_root = files(_TEMPLATE_PACKAGE) + for relative_path, template_name in _FLAT_FILES.items(): + target = project_root / relative_path + existed_before = target.exists() + if existed_before and not force: + result.skipped_files.append(target) + continue + + target.parent.mkdir(parents=True, exist_ok=True) + if not target.parent.exists(): + result.created_dirs.append(target.parent) + + content = templates_root.joinpath(template_name).read_text(encoding="utf-8") + target.write_text(content, encoding="utf-8") + + if existed_before: + result.overwritten_files.append(target) + else: + result.created_files.append(target) + + return result diff --git a/src/agentops/templates/agentops.yaml b/src/agentops/templates/agentops.yaml new file mode 100644 index 00000000..71154f4b --- /dev/null +++ b/src/agentops/templates/agentops.yaml @@ -0,0 +1,26 @@ +# AgentOps configuration — 1.0 flat schema. +# +# The four required fields are 'version', 'agent', 'dataset', and (optionally) +# 'thresholds'. AgentOps infers evaluators from the agent type and dataset +# columns, so most users only need this much. +# +# Examples: +# +# agent: "my-rag:3" # Foundry prompt agent (name:version) +# agent: "https://...foundry.../agents/" # Foundry hosted endpoint +# agent: "https://api.example.com/chat" # any HTTP/JSON agent (ACA, AKS, custom) +# agent: "model:gpt-4o" # raw Foundry model deployment + +version: 1 + +agent: "my-agent:1" + +dataset: .agentops/data/smoke.jsonl + +# Optional. Override the auto-selected pass/fail thresholds. AgentOps fills in +# sensible defaults for the metrics that are auto-selected. +# +# thresholds: +# coherence: ">=3" +# groundedness: ">=3" +# avg_latency_seconds: "<=10" diff --git a/src/agentops/templates/smoke.jsonl b/src/agentops/templates/smoke.jsonl new file mode 100644 index 00000000..b2246374 --- /dev/null +++ b/src/agentops/templates/smoke.jsonl @@ -0,0 +1,3 @@ +{"input": "What is AgentOps?", "expected": "AgentOps is a CLI for evaluating Foundry agents."} +{"input": "Which formats does it produce?", "expected": "It writes results.json and report.md."} +{"input": "How do I configure thresholds?", "expected": "Use the 'thresholds' map in agentops.yaml."} diff --git a/src/agentops/templates/workflows/agentops-eval.yml b/src/agentops/templates/workflows/agentops-eval.yml index 580e4315..4fcdc74d 100644 --- a/src/agentops/templates/workflows/agentops-eval.yml +++ b/src/agentops/templates/workflows/agentops-eval.yml @@ -24,9 +24,9 @@ on: workflow_dispatch: inputs: config: - description: "Path to run.yaml (default: .agentops/run.yaml)" + description: "Path to agentops.yaml or run.yaml (default: agentops.yaml)" required: false - default: ".agentops/run.yaml" + default: "agentops.yaml" output: description: "Output directory for results" required: false @@ -77,7 +77,14 @@ jobs: - name: Resolve config path id: config run: | - CONFIG="${{ github.event.inputs.config || '.agentops/run.yaml' }}" + CONFIG="${{ github.event.inputs.config }}" + if [ -z "$CONFIG" ]; then + if [ -f "agentops.yaml" ]; then + CONFIG="agentops.yaml" + else + CONFIG=".agentops/run.yaml" + fi + fi echo "path=$CONFIG" >> "$GITHUB_OUTPUT" - name: Resolve output directory diff --git a/tests/integration/test_cli_flat_schema.py b/tests/integration/test_cli_flat_schema.py new file mode 100644 index 00000000..2dcc16f4 --- /dev/null +++ b/tests/integration/test_cli_flat_schema.py @@ -0,0 +1,119 @@ +"""CLI tests for the 1.0 flat schema path on ``agentops eval run``.""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from agentops.cli.app import app + +runner = CliRunner() + + +class _EchoHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8")) + message = body.get("message", "") + payload = json.dumps({"text": f"echo: {message}"}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args, **kwargs) -> None: # noqa: D401 + pass + + +@pytest.fixture() +def echo_server(): + server = HTTPServer(("127.0.0.1", 0), _EchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}/chat" + finally: + server.shutdown() + thread.join(timeout=1) + + +def _write_dataset(path: Path) -> None: + rows = [ + {"input": "say hi", "expected": "hi"}, + {"input": "say bye", "expected": "bye"}, + ] + path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + +def _write_flat_config(path: Path, *, agent: str, dataset: Path) -> None: + payload = { + "version": 1, + "agent": agent, + "dataset": str(dataset), + "evaluators": [{"name": "F1ScoreEvaluator"}], + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_eval_run_routes_flat_schema_to_pipeline( + tmp_path: Path, echo_server: str +) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config = tmp_path / "agentops.yaml" + _write_flat_config(config, agent=echo_server, dataset=dataset) + output = tmp_path / "out" + + result = runner.invoke( + app, + [ + "eval", + "run", + "--config", + str(config), + "--output", + str(output), + ], + ) + + assert result.exit_code in (0, 2), result.output + assert (output / "results.json").exists() + assert (output / "report.md").exists() + + +def test_eval_run_supports_baseline_flag(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config = tmp_path / "agentops.yaml" + _write_flat_config(config, agent=echo_server, dataset=dataset) + + baseline_dir = tmp_path / "baseline" + runner.invoke( + app, + ["eval", "run", "--config", str(config), "--output", str(baseline_dir)], + ) + current = tmp_path / "current" + result = runner.invoke( + app, + [ + "eval", + "run", + "--config", + str(config), + "--output", + str(current), + "--baseline", + str(baseline_dir / "results.json"), + ], + ) + assert result.exit_code in (0, 2), result.output + payload = json.loads((current / "results.json").read_text(encoding="utf-8")) + assert payload["comparison"] is not None + assert payload["comparison"]["baseline_path"].endswith("results.json") diff --git a/tests/unit/test_init_flat.py b/tests/unit/test_init_flat.py new file mode 100644 index 00000000..e99f082e --- /dev/null +++ b/tests/unit/test_init_flat.py @@ -0,0 +1,42 @@ +"""Tests for ``initialize_flat_workspace``.""" + +from __future__ import annotations + +from pathlib import Path + +from agentops.services.initializer import initialize_flat_workspace + + +def test_creates_minimal_layout(tmp_path: Path) -> None: + result = initialize_flat_workspace(tmp_path, force=False) + + config = tmp_path / "agentops.yaml" + dataset = tmp_path / ".agentops" / "data" / "smoke.jsonl" + + assert config.exists() + assert dataset.exists() + assert config in result.created_files + assert dataset in result.created_files + assert "version: 1" in config.read_text(encoding="utf-8") + + +def test_skips_existing_files_without_force(tmp_path: Path) -> None: + initialize_flat_workspace(tmp_path, force=False) + config = tmp_path / "agentops.yaml" + config.write_text("custom\n", encoding="utf-8") + + result = initialize_flat_workspace(tmp_path, force=False) + + assert config in result.skipped_files + assert config.read_text(encoding="utf-8") == "custom\n" + + +def test_force_overwrites(tmp_path: Path) -> None: + initialize_flat_workspace(tmp_path, force=False) + config = tmp_path / "agentops.yaml" + config.write_text("custom\n", encoding="utf-8") + + result = initialize_flat_workspace(tmp_path, force=True) + + assert config in result.overwritten_files + assert "version: 1" in config.read_text(encoding="utf-8") From 7f0e435ca52fbc3b56a1beaa4d40d3db0694cc8a Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 27 Apr 2026 11:27:43 -0300 Subject: [PATCH 04/70] docs(readme): remove legacy multi-file quickstart section Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 57 ------------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/README.md b/README.md index 0b3bba96..0dca814f 100644 --- a/README.md +++ b/README.md @@ -110,63 +110,6 @@ The report grows a `Comparison vs Baseline` section with per-metric deltas. --- -## Quickstart (legacy multi-file layout) - -

-Quickstart demo: agentops init and eval run -

- -### 1) Install - -```bash -python -m venv .venv -# activate your venv in the current shell -python -m pip install -U pip -python -m pip install agentops-toolkit -``` - -### 2) Initialize and Configure - -```bash -agentops init -``` - -This creates `.agentops/` with starter bundles, datasets, and run configs for common scenarios (model quality, RAG, agent workflow, content safety). - -Set your Foundry project endpoint: - -```bash -export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" -``` - -Then edit `.agentops/run.yaml` to set your `agent_id` and `model` deployment name. - -> Authentication uses `DefaultAzureCredential` — run `az login` locally, or use service principal env vars in CI. - -### 3) Run Evaluation - -```bash -agentops eval run -``` - -Results are written to `.agentops/results/latest/`: -- `results.json` — machine-readable scores -- `report.md` — human-readable summary - -To run a different scenario: - -```bash -agentops eval run --config .agentops/run-rag.yaml -``` - -To regenerate the report from existing results: - -```bash -agentops report generate -``` - -See [Concepts](https://github.com/Azure/agentops/blob/main/docs/concepts.md) for an overview of bundles, datasets, evaluators, backends, and the configuration model. - ## Commands | Command | Description | Status | From ff62b79fdc977af46316394bac535d368e9d8fff Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 27 Apr 2026 11:31:15 -0300 Subject: [PATCH 05/70] docs: drop unimplemented commands and stale planning docs from README - Remove tracing/monitoring/run-history bullets and 'planned' commands rows - Drop version markers from headings and tutorial filename - Trim documentation links to implemented topics - Delete obsolete analysis-issue-51 and run-yaml-schema files - Delete telemetry doc (tracing not yet implemented) - Rename tutorial-1.0-quickstart.md -> tutorial-quickstart.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 65 +-- docs/analysis-issue-51-cicd-field-insights.md | 445 ----------------- docs/analysis-issue-51-two-track.md | 447 ------------------ docs/media/agentops-diagrams.vsdx | Bin 0 -> 1958863 bytes docs/run-yaml-schema.md | 274 ----------- docs/telemetry.md | 446 ----------------- ...0-quickstart.md => tutorial-quickstart.md} | 4 +- 7 files changed, 16 insertions(+), 1665 deletions(-) delete mode 100644 docs/analysis-issue-51-cicd-field-insights.md delete mode 100644 docs/analysis-issue-51-two-track.md create mode 100644 docs/media/agentops-diagrams.vsdx delete mode 100644 docs/run-yaml-schema.md delete mode 100644 docs/telemetry.md rename docs/{tutorial-1.0-quickstart.md => tutorial-quickstart.md} (95%) diff --git a/README.md b/README.md index 0dca814f..07e510fb 100644 --- a/README.md +++ b/README.md @@ -19,24 +19,15 @@ AgentOps CLI for evaluation, observability, and operational workflows for Micros ## Overview -AgentOps Toolkit is a CLI built on Microsoft Foundry that standardizes evaluation and operational workflows for AI agents and models, helping teams run, monitor, and automate AgentOps processes. +AgentOps Toolkit is a CLI built on Microsoft Foundry that standardizes evaluation workflows for AI agents and models, helping teams run and automate evaluations with consistent inputs and outputs. The project enables: - Consistent local and CI execution of agent evaluations -- Reusable evaluation policies through bundles -- Operational observability through tracing, monitoring, and run inspection +- Automatic evaluator selection based on dataset shape (RAG, agent-with-tools, model quality) - Stable machine-readable outputs for automation - Human-readable reports for PR reviews and quality gates - -Operational capabilities include: - -- Standardized evaluation workflows -- Run history and result inspection -- Tracing and observability -- Monitoring (dashboards and alerts) -- CI/CD automation -- Operational reporting and analysis +- Baseline comparison to detect regressions across runs Core outputs: @@ -49,9 +40,7 @@ Exit code contract: - `2` execution succeeded but one or more thresholds failed - `1` runtime or configuration error -## Quickstart (1.0) - -> The 1.0 release introduces a flat, three-line config for the most common case. The legacy multi-file workspace is still supported — see [`docs/how-it-works.md`](docs/how-it-works.md) for details. +## Quickstart ### 1) Install @@ -112,47 +101,21 @@ The report grows a `Comparison vs Baseline` section with per-metric deltas. ## Commands -| Command | Description | Status | -|---|---|---| -| `agentops --version` | Show installed version | ✅ | -| `agentops init [--path DIR]` | Scaffold project workspace, starter files, and coding agent skills | ✅ | -| `agentops eval run [--config PATH]` | Evaluate a dataset against a bundle | ✅ | -| `agentops eval compare --runs ID1,ID2` | Compare two past runs | ✅ | -| `agentops report generate [--in FILE]` | Regenerate `report.md` from `results.json` | ✅ | -| `agentops workflow generate` | Generate GitHub Actions workflow | ✅ | -| `agentops skills install [--platform

]` | Install coding agent skills (Copilot, Claude) | ✅ | -| `agentops run list\|show` | List or inspect past runs | 🚧 | -| `agentops bundle list\|show` | Browse bundle catalog | 🚧 | -| `agentops dataset validate\|describe` | Dataset utilities | 🚧 | -| `agentops trace init` | Tracing setup | 🚧 | -| `agentops monitor setup\|show\|configure` | Monitoring operations | 🚧 | - -Planned commands return a friendly message indicating they are not yet implemented. +| Command | Description | +|---|---| +| `agentops --version` | Show installed version | +| `agentops init --flat` | Bootstrap `agentops.yaml` and a seed dataset | +| `agentops eval run [--config PATH] [--baseline PATH]` | Run an evaluation | +| `agentops report generate [--in FILE]` | Regenerate `report.md` from `results.json` | +| `agentops workflow generate` | Generate GitHub Actions workflow | +| `agentops skills install [--platform

]` | Install coding agent skills (Copilot, Claude) | ## Documentation -### Concepts and Architecture - -- [Concepts](https://github.com/Azure/agentops/blob/main/docs/concepts.md) — bundles, datasets, evaluators, backends, configuration model -- [How It Works](https://github.com/Azure/agentops/blob/main/docs/how-it-works.md) — architecture, request flow, full schema reference -- [Bundles](https://github.com/Azure/agentops/blob/main/docs/bundles.md) — bundle authoring and evaluator configuration - -### Tutorials - -- [Model-direct evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-model-direct.md) -- [Foundry agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-basic-foundry-agent.md) -- [RAG evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-rag.md) -- [HTTP-deployed agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-http-agent.md) -- [Conversational agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-conversational-agent.md) -- [Agent workflow evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-agent-workflow.md) -- [Baseline comparison](https://github.com/Azure/agentops/blob/main/docs/tutorial-baseline-comparison.md) - -### Operations - +- [Quickstart tutorial](https://github.com/Azure/agentops/blob/main/docs/tutorial-quickstart.md) - [CI/CD with GitHub Actions](https://github.com/Azure/agentops/blob/main/docs/ci-github-actions.md) -- [Copilot skills installation](https://github.com/Azure/agentops/blob/main/docs/tutorial-copilot-skills.md) -- [Release process](https://github.com/Azure/agentops/blob/main/docs/release-process.md) - [Built-in evaluator reference](https://github.com/Azure/agentops/blob/main/docs/foundry-evaluation-sdk-built-in-evaluators.md) +- [Release process](https://github.com/Azure/agentops/blob/main/docs/release-process.md) ## Contributing diff --git a/docs/analysis-issue-51-cicd-field-insights.md b/docs/analysis-issue-51-cicd-field-insights.md deleted file mode 100644 index 36e51e2a..00000000 --- a/docs/analysis-issue-51-cicd-field-insights.md +++ /dev/null @@ -1,445 +0,0 @@ -# Issue #51 — Review CI/CD Based on Field Insights - -**Date:** 2026-04-03 -**Issue:** https://github.com/Azure/agentops/issues/51 -**Author:** placerda -**Reference repo:** https://github.com/hrprtkaur88/foundrycicdbasic - ---- - -## 1. Executive Summary - -This analysis evaluates how well AgentOps Toolkit serves as a CI/CD-ready -evaluation tool based on real-world pipeline patterns observed in Harpreet's -Foundry CI/CD reference repository. The goal is to identify what prevents teams -like Harpreet's from replacing their custom Python scripts with -`agentops eval run`, and what AgentOps must improve to be viable in real -CI/CD environments. - -**Key finding:** AgentOps has strong CI/CD foundations (exit codes, artifacts, -declarative config, generated workflow) but is missing critical evaluator -coverage and data-source patterns that real-world pipelines require. A team -using Harpreet's pipeline today cannot switch to AgentOps without losing -evaluator coverage. - ---- - -## 2. Task Analysis - -### Task 1: Review Harpreet repository and pipeline structure - -**What the repo is:** -A reference implementation showing how to create, test, evaluate, and red-team -Foundry agents using raw Python scripts orchestrated by CI/CD pipelines. - -**Repository structure:** - -``` -foundrycicdbasic/ -├── createagent.py # Creates a Foundry agent via Agent Framework SDK -├── exagent.py # Smoke-tests an existing agent with a real query -├── agenteval.py # Runs cloud evaluation via OpenAI Evals API -├── agenteval_classic.py # Local evaluation fallback -├── redteam.py # Red-team safety evaluation -├── redteam_classic.py # Red-team local fallback -├── requirements.txt # Unpinned runtime dependencies -├── sample.env # Example environment variables -├── data_folder/ # Red-team taxonomy + output files -├── .github/workflows/ -│ ├── create-agent-multi-env.yml # GitHub Actions: deploy agent (dev→test→prod) -│ └── agent-consumption-multi-env.yml # GitHub Actions: test→eval→redteam (dev→test→prod) -├── cicd/ -│ ├── createagentpipeline.yml # Azure DevOps: deploy agent -│ └── agentconsumptionpipeline.yml # Azure DevOps: test→eval→redteam -└── cicd_patterns/ - └── foundry-cicd-workflow.pptx # Presentation on patterns -``` - -**Pipeline flow (agent-consumption-multi-env.yml):** - -``` -build (validate syntax) - → test-dev (exagent.py — smoke-test agent) - → evaluate-test (agenteval.py — cloud evaluation) - → red-team-test (redteam.py — safety evaluation) - → verify-prod (exagent.py — production verification) -``` - -**Key observations:** - -1. **All evaluation logic is imperative** — evaluator names, data mappings, - test data, and testing criteria are hardcoded in Python scripts. -2. **No thresholds or gating** — every eval/redteam step uses - `continue-on-error: true`. The pipeline never blocks on quality. -3. **Authentication uses service principal JSON blobs** — stored as - `AZURE_CREDENTIALS_*` secrets, not OIDC. -4. **Dual platform** — same pipelines exist for both GitHub Actions and - Azure DevOps (manually duplicated). -5. **Inline test data** — `agenteval.py` has query/response/tool_definitions - hardcoded in the script, not in external data files. - -### Task 2: Identify evaluation patterns used in real scenarios - -The following evaluation patterns are used in Harpreet's pipeline. Each is -mapped to AgentOps support status. - -#### Pattern A: Agent smoke test (exagent.py) - -**What it does:** Retrieves an existing agent by name, sends a real query, -handles MCP approval requests, and prints the response with citations. - -**Purpose in CI/CD:** Validates the agent is alive and responsive before -running expensive evaluations. - -**AgentOps equivalent:** None. AgentOps has no "health check" or "smoke test" -concept. The `agentops eval run` command goes straight to evaluation. - -**Gap severity:** Low. This is a convenience — users can add a custom step -before `agentops eval run` in their pipeline. - -#### Pattern B: Cloud evaluation with inline data (agenteval.py) - -**What it does:** -1. Creates an OpenAI client from the Foundry project client -2. Defines `data_source_config` with `type: custom` and an item schema -3. Defines `testing_criteria` — a list of `azure_ai_evaluator` entries -4. Calls `client.evals.create()` to create an eval group -5. Calls `client.evals.runs.create()` with inline JSONL data -6. Polls until completion -7. Retrieves output items - -**Evaluators used:** - -| Category | Evaluator | Builtin name | AgentOps support | -|---|---|---|---| -| System | Task Completion | `builtin.task_completion` | **Not supported** | -| System | Task Adherence | `builtin.task_adherence` | **Not supported** | -| System | Intent Resolution | `builtin.intent_resolution` | **Not supported** | -| RAG | Groundedness | `builtin.groundedness` | Supported | -| RAG | Relevance | `builtin.relevance` | **Not supported** | -| Process | Tool Call Accuracy | `builtin.tool_call_accuracy` | Supported | -| Process | Tool Selection | `builtin.tool_selection` | **Not supported** | -| Process | Tool Input Accuracy | `builtin.tool_input_accuracy` | **Not supported** | -| Process | Tool Output Utilization | `builtin.tool_output_utilization` | **Not supported** | - -**Data format used:** -- `query`: array of message objects (system + user messages) -- `response`: array of message objects (assistant + tool_call + tool_result) -- `tool_definitions`: array of tool schemas -- `tool_calls`: null (derived from response) - -**AgentOps data format:** -- `input`: string (simple text field from JSONL) -- `expected`: string (simple text field from JSONL) -- `context`: optional string - -**Gap severity:** **Critical.** 7 of 9 evaluators used in the field are not -supported by AgentOps. The data format is also incompatible — Harpreet uses -conversation-format arrays while AgentOps expects simple string fields. - -#### Pattern C: Red-team / safety evaluation (redteam.py) - -**What it does:** -1. Creates an agent version via `project_client.agents.create_version()` -2. Defines safety testing criteria: - - `builtin.prohibited_actions` - - `builtin.task_adherence` - - `builtin.sensitive_data_leakage` - - `builtin.self_harm` - - `builtin.violence` - - `builtin.sexual` - - `builtin.hate_unfairness` -3. Creates evaluation taxonomy via `project_client.evaluation_taxonomies.create()` -4. Creates eval run with `data_source.type: azure_ai_red_team` -5. Uses `attack_strategies: ["Flip", "Base64"]` with generated adversarial inputs -6. Polls until completion, saves results to JSON - -**AgentOps equivalent:** None. AgentOps has no concept of: -- Red-team data sources (`azure_ai_red_team`) -- Safety evaluators (prohibited_actions, sensitive_data_leakage, violence, etc.) -- Attack strategies -- Evaluation taxonomies - -**Gap severity:** **High.** Red-team testing is a major field requirement. -However, this may be better addressed as a separate `agentops redteam` command -rather than extending `agentops eval run`, since the data source model is -fundamentally different (generated adversarial inputs vs. user-provided JSONL). - -#### Pattern D: Multi-environment sequential deployment - -**What it does:** Runs the same scripts across dev → test → prod environments, -with each stage depending on the previous. Production requires manual approval -via GitHub Environment protection rules. - -**AgentOps equivalent:** Not directly relevant to the AgentOps tool — this is -a pipeline orchestration pattern. AgentOps's `project_endpoint_env` config -already supports being called in different environments by varying the -endpoint secret. No tool change needed. - -**Gap severity:** None for the tool. Documentation gap only. - -#### Pattern E: Scheduled security scans - -**What it does:** Weekly cron trigger (`0 2 * * 1`) runs the full -test → eval → redteam pipeline on Monday mornings. - -**AgentOps equivalent:** Not relevant to the tool — this is a pipeline trigger -pattern. `agentops eval run` works fine when invoked by a cron job. - -**Gap severity:** None for the tool. Documentation gap only. - -### Task 3: Define supported CI/CD integration models - -Based on field analysis, AgentOps should support these integration models: - -| Model | Description | Tool readiness | -|---|---|---| -| **PR gating** | `agentops eval run` in a PR workflow; exit code 2 blocks merge | **Ready** — implemented and documented | -| **Scheduled regression** | Cron-triggered eval run to detect drift | **Ready** — CLI works, needs documentation | -| **Post-deployment validation** | Run eval after deploying to an environment | **Ready** — CLI works, needs documentation | -| **Multi-config matrix** | Run multiple eval configs in parallel | **Ready** — documented with matrix strategy | -| **Advisory mode** | Run eval and report results without blocking | **Partially ready** — exit code 2 blocks; no `--no-fail` flag | - -### Task 4: Define best practices for gating deployments based on evaluations - -**What AgentOps provides today:** - -| Capability | Status | Evidence | -|---|---|---| -| Exit code contract (0/1/2) | Implemented | `cli/app.py` raises `typer.Exit(code=2)` on threshold failure | -| Declarative thresholds in YAML | Implemented | `bundles/*.yaml` with `thresholds[]` | -| Per-metric threshold criteria | Implemented | `>=`, `>`, `<=`, `<`, `==`, `true`/`false` in `thresholds.py` | -| Per-row threshold evaluation | Implemented | `runner.py` `_evaluate_item_thresholds()` | -| PR comment with report | Implemented | Workflow template posts/updates PR comment | -| Job summary | Implemented | Workflow writes to `$GITHUB_STEP_SUMMARY` | -| Artifacts on failure | Implemented | `if: always()` on artifact upload step | - -**What's missing for real-world gating:** - -| Gap | Impact | -|---|---| -| No `--no-fail` / `--advisory` flag | Teams can't run eval in "observe only" mode (like Harpreet's `continue-on-error`) | -| `agentops config validate` not implemented | Teams can't fail-fast on bad config before running expensive evaluations | -| No threshold on safety evaluators | Can't gate on red-team results since safety evaluators aren't supported | - -### Task 5: Identify gaps in current CLI for CI/CD usage - -| Gap | Category | Severity | Detail | -|---|---|---|---| -| Missing cloud evaluators | Evaluator coverage | **Critical** | 7 of 9 evaluators used in field are unsupported: `task_completion`, `task_adherence`, `intent_resolution`, `relevance`, `tool_selection`, `tool_input_accuracy`, `tool_output_utilization` | -| No conversation-format data | Data model | **High** | Field uses array-of-messages for query/response; AgentOps only supports simple string fields | -| No red-team support | Feature | **High** | No safety evaluators, no `azure_ai_red_team` data source, no attack strategies | -| No `--no-fail` flag | CLI | **Medium** | Can't run in advisory mode without `continue-on-error` in the pipeline YAML | -| `config validate` not implemented | CLI | **Medium** | Can't pre-validate configs in CI before running eval | -| `dataset validate` not implemented | CLI | **Medium** | Can't verify dataset integrity in CI | -| No Azure DevOps template | Documentation | **Low** | `agentops config cicd` only generates GitHub Actions; ADO users must write their own | - ---- - -## 3. Acceptance Criteria Assessment - -### AC 1: CI/CD integration patterns are clearly defined - -**Verdict: PARTIALLY MET** - -**What exists:** -- `docs/ci-github-actions.md` — comprehensive guide covering triggers, auth, - exit codes, artifacts, PR comments, job summary, troubleshooting -- Generated workflow template via `agentops config cicd` -- Matrix strategy documentation for multi-config runs -- Internal CI/CD workflows documented for contributors - -**What's missing:** -- No documentation for Azure DevOps integration -- No documentation for "advisory mode" (run without gating) -- No documentation for scheduled evaluation pattern -- The patterns are defined for the *simple case* (model-direct with similarity) - but not for the *real-world case* (agent evaluation with process/system - evaluators) - -**To close:** Document Azure DevOps integration pattern. Document advisory -mode. Ensure patterns cover agent evaluation scenarios, not just model-direct. - -### AC 2: Pipelines support evaluation as a gating mechanism - -**Verdict: MET (for supported evaluators)** - -**Evidence:** -- Exit code 0/1/2 contract is implemented and tested -- Workflow template uses `exit $EXIT_CODE` — non-zero fails the job -- Threshold evaluation supports multiple criteria operators -- Per-row and aggregate threshold evaluation is implemented -- CLI propagates exit code 2 via `raise typer.Exit(code=2)` - -**Caveat:** Gating only works for the evaluators AgentOps supports. Since most -field-used evaluators are unsupported, the gating mechanism exists but can't -be applied to the metrics teams actually care about (task_completion, -intent_resolution, etc.). - -### AC 3: Exit codes are correctly interpreted in CI/CD - -**Verdict: MET** - -**Evidence:** -- Workflow template maps exit codes to step summary messages - (0 → pass, 2 → threshold fail, else → error) -- Exit code saved to `$GITHUB_OUTPUT` for downstream consumption -- `test_cicd.py` asserts `EXIT_CODE` and `exit $EXIT_CODE` are in template -- GitHub Actions natively fails on non-zero — no special handling needed -- Exit code semantics documented in `docs/ci-github-actions.md` - -### AC 4: Artifacts are generated and usable in pipeline context - -**Verdict: MET** - -**Evidence:** -- Workflow uploads 6 artifact files: `results.json`, `report.md`, - `backend_metrics.json`, `cloud_evaluation.json`, `backend.stdout.log`, - `backend.stderr.log` -- Upload uses `if: always()` — artifacts available even on failure -- `results.json` has versioned Pydantic schema — machine-readable -- `report.md` is human-readable and posted as PR comment -- `cloud_evaluation.json` includes `report_url` for Foundry portal deep-link -- `agentops report --in results.json` can regenerate reports from artifacts - -### AC 5: At least one reference pipeline is documented - -**Verdict: MET** - -**Evidence:** -- `docs/ci-github-actions.md` is a complete reference pipeline guide -- `agentops config cicd` generates a tested, ready-to-use workflow -- Template includes inline comments explaining every step -- Quick start, auth setup, customization, and troubleshooting covered - -### AC 6: Integration works with real-world scenarios - -**Verdict: NOT MET** - -**Evidence from field analysis:** - -Harpreet's pipeline represents a real-world scenario. To replace their -`agenteval.py` with `agentops eval run`, a user would need to: - -1. **Define evaluators in a bundle YAML** — but 7 of 9 evaluators they use - are not supported by AgentOps -2. **Provide test data in JSONL** — but the field uses conversation-format - arrays (query as message list, response as message list with tool calls), - while AgentOps expects simple string fields -3. **Get evaluation results** — AgentOps produces `results.json` and - `report.md`, which is better than Harpreet's raw stdout, but the results - won't contain the metrics teams need -4. **Gate on results** — AgentOps has threshold gating, which Harpreet's - pipeline lacks, but it can only gate on supported evaluators - -**What a user would need to do today to use AgentOps in Harpreet's pipeline:** - -```yaml -# What they want to write: -bundle: - evaluators: - - name: TaskCompletionEvaluator # ❌ not supported - - name: TaskAdherenceEvaluator # ❌ not supported - - name: IntentResolutionEvaluator # ❌ not supported - - name: GroundednessEvaluator # ✅ supported - - name: RelevanceEvaluator # ❌ not supported - - name: ToolCallAccuracyEvaluator # ✅ supported - - name: ToolSelectionEvaluator # ❌ not supported - -# What they can actually use today: -bundle: - evaluators: - - name: GroundednessEvaluator # ✅ - - name: ToolCallAccuracyEvaluator # ✅ - # ...that's it -``` - -**Blockers preventing real-world adoption:** - -| Blocker | Why it blocks | -|---|---| -| Missing evaluators | Teams can't measure what matters to them | -| String-only data format | Teams can't provide conversation-format test data | -| No red-team | Teams must maintain a separate `redteam.py` alongside AgentOps | - ---- - -## 4. Gap Prioritization for Closing the Issue - -### Priority 1 — Critical (blocks AC 6) - -| Item | What to do | Effort | -|---|---|---| -| Add system evaluators | Add `task_completion`, `task_adherence`, `intent_resolution` to `_cloud_evaluator_data_mapping` | Low — mapping only, no new API calls | -| Add RAG evaluator: relevance | Add `relevance` alongside existing `groundedness` | Low | -| Add process evaluators | Add `tool_selection`, `tool_input_accuracy`, `tool_output_utilization` to `_EVALUATORS_NEEDING_TOOL_CALLS` or a new set | Low-Medium — need to verify data_mapping for each | - -These evaluators all use the same `azure_ai_evaluator` type and -`builtin.` pattern that AgentOps already supports. The gap is in the -`_cloud_evaluator_data_mapping` function, which doesn't know how to build -`data_mapping` for these evaluators. Each new evaluator needs: -- An entry in the appropriate frozenset (or a new one) -- The correct `data_mapping` fields (query, response, tool_calls, tool_definitions, etc.) - -### Priority 2 — High (improves real-world viability) - -| Item | What to do | Effort | -|---|---|---| -| Conversation-format data support | Allow JSONL rows with array-of-messages for query/response fields | Medium — requires dataset format model changes | -| `--no-fail` / `--advisory` flag | Add CLI flag that makes exit code always 0 (report thresholds but don't gate) | Low | -| `config validate` command | Implement the planned command to pre-validate configs in CI | Medium | - -### Priority 3 — Medium (documentation) - -| Item | What to do | Effort | -|---|---|---| -| Azure DevOps integration pattern | Document how to use `agentops eval run` in an ADO pipeline | Low — docs only | -| Scheduled evaluation pattern | Document cron-triggered eval for drift detection | Low — docs only | -| Advisory mode pattern | Document how to run eval without gating (once `--no-fail` exists) | Low — docs only | -| Multi-environment pattern | Document how to use `project_endpoint_env` across environments | Low — docs only | - -### Priority 4 — Future (separate feature) - -| Item | What to do | Effort | -|---|---|---| -| Red-team support | New command or new data source type — fundamentally different flow | High — new feature | -| Safety evaluators | `prohibited_actions`, `sensitive_data_leakage`, `violence`, etc. | Medium — requires red-team data source | - ---- - -## 5. Recommendation - -**To close issue #51, focus on Priority 1 (missing evaluators).** This is the -single biggest blocker for real-world CI/CD adoption. The evaluators all follow -the same `azure_ai_evaluator` / `builtin.` pattern that AgentOps already -implements — the gap is mechanical, not architectural. - -Adding 7 evaluators to `foundry_backend.py` would change the AC 6 verdict from -"NOT MET" to "PARTIALLY MET" (still missing conversation-format data and -red-team, but the core evaluation flow would work for the majority of -field-used evaluators). - -Red-team support (Priority 4) should be tracked as a separate issue — it -requires a different data source model (`azure_ai_red_team` with attack -strategies and taxonomy generation) that doesn't fit the current -`agentops eval run` flow. - ---- - -## 6. Summary Scorecard - -| Acceptance Criterion | Verdict | -|---|---| -| AC 1: CI/CD integration patterns clearly defined | ⚠️ Partially met | -| AC 2: Pipelines support evaluation as gating mechanism | ✅ Met | -| AC 3: Exit codes correctly interpreted in CI/CD | ✅ Met | -| AC 4: Artifacts generated and usable in pipeline context | ✅ Met | -| AC 5: At least one reference pipeline documented | ✅ Met | -| AC 6: Integration works with real-world scenarios | ❌ Not met | - -**Overall: 4/6 met, 1/6 partially met, 1/6 not met.** - -The blocking gap is evaluator coverage. AgentOps has the right architecture -for CI/CD integration — declarative config, exit-code gating, artifact -production, generated workflows — but it cannot evaluate the metrics that -real-world Foundry agent pipelines need. diff --git a/docs/analysis-issue-51-two-track.md b/docs/analysis-issue-51-two-track.md deleted file mode 100644 index b320c71e..00000000 --- a/docs/analysis-issue-51-two-track.md +++ /dev/null @@ -1,447 +0,0 @@ -# Issue #51 — Two-Track Analysis - -**Date:** 2026-04-03 - ---- - -## Track 1: How to Fully Support Foundry Default Evaluators - -### Current Architecture - -The cloud evaluation path in `foundry_backend.py` builds evaluators like this: - -```python -builtin_name = _to_builtin_evaluator_name(evaluator.name) # "SimilarityEvaluator" → "similarity" -criterion = { - "type": "azure_ai_evaluator", - "name": evaluator.name, - "evaluator_name": f"builtin.{builtin_name}", - "data_mapping": _cloud_evaluator_data_mapping(builtin_name, input_field, expected_field, context_field), -} -if _cloud_evaluator_needs_model(builtin_name): - criterion["initialization_parameters"] = {"deployment_name": settings.model} -``` - -The `_cloud_evaluator_data_mapping` function routes evaluators to the correct -`data_mapping` based on frozenset membership: - -``` -default path → {"query": "{{item.X}}", "response": "{{sample.output_text}}"} -_NLP_ONLY_EVALUATORS → no "query", just "response" -_GROUND_TRUTH → adds "ground_truth": "{{item.Y}}" -_CONTEXT → adds "context": "{{item.Z}}" -_TOOL_CALLS → adds "tool_calls": "{{sample.tool_calls}}", "tool_definitions": "{{item.tool_definitions}}" -``` - -### Problem: Only 8 of ~35 evaluators are routed correctly - -Any evaluator NOT in any frozenset falls to the default path (`query` + `response`). -This accidentally works for some evaluators (like `coherence`) but silently sends -wrong data_mappings for many others. - -### What Each Evaluator Actually Needs - -Based on Foundry cloud evaluation docs (2026-04-02), here are the correct -`data_mapping` patterns for every built-in evaluator: - -#### Pattern 1: query + response (simplest — default path) - -Works with current default path. No code change needed. - -| Evaluator | builtin name | Needs model | Status | -|---|---|---|---| -| CoherenceEvaluator | `coherence` | Yes | ✅ Works today (falls to default) | -| FluencyEvaluator | `fluency` | Yes | ✅ Works today | -| RelevanceEvaluator | `relevance` | Yes | ✅ Works today | -| IntentResolutionEvaluator | `intent_resolution` | Yes | ✅ Works today | -| TaskCompletionEvaluator | `task_completion` | Yes | ✅ Works today | -| ViolenceEvaluator | `violence` | Yes | ✅ Works today | -| SexualEvaluator | `sexual` | Yes | ✅ Works today | -| SelfHarmEvaluator | `self_harm` | Yes | ✅ Works today | -| HateUnfairnessEvaluator | `hate_unfairness` | Yes | ✅ Works today | -| ContentSafetyEvaluator | `content_safety` | Yes | ✅ Works today | -| ProtectedMaterialEvaluator | `protected_material` | Yes | ✅ Works today | -| CodeVulnerabilityEvaluator | `code_vulnerability` | Yes | ✅ Works today | -| UngroundedAttributesEvaluator | `ungrounded_attributes` | Yes | ✅ Works today | -| IndirectAttackEvaluator | `indirect_attack` | Yes | ✅ Works today | - -**Verdict:** These 14 evaluators already work with the current code — users -just don't know they can use them because they're not documented/tested. - -#### Pattern 2: query + response (output_items) — agent structured output - -`task_adherence` needs `{{sample.output_items}}` instead of -`{{sample.output_text}}` for the response field, because it needs to see the -full structured agent output (tool calls, intermediate steps). - -| Evaluator | builtin name | response field | Status | -|---|---|---|---| -| TaskAdherenceEvaluator | `task_adherence` | `{{sample.output_items}}` | ❌ **Broken** — sends `output_text` | - -**Fix required:** Add `task_adherence` to a new set -`_EVALUATORS_NEEDING_OUTPUT_ITEMS` and map `response` to -`{{sample.output_items}}` instead of `{{sample.output_text}}`. - -#### Pattern 3: response + ground_truth (existing) - -Already implemented via `_EVALUATORS_NEEDING_GROUND_TRUTH`. - -| Evaluator | builtin name | Status | -|---|---|---| -| SimilarityEvaluator | `similarity` | ✅ Supported | -| ResponseCompletenessEvaluator | `response_completeness` | ❌ Missing from frozenset | - -**Fix required:** Add `response_completeness` to `_EVALUATORS_NEEDING_GROUND_TRUTH`. - -#### Pattern 4: NLP only — no query, no model (existing) - -Already implemented via `_NLP_ONLY_EVALUATORS`. - -| Evaluator | builtin name | Status | -|---|---|---| -| F1ScoreEvaluator | `f1_score` | ✅ Supported | -| BleuScoreEvaluator | `bleu` | ✅ Supported | -| GleuScoreEvaluator | `gleu` | ✅ Supported | -| RougeScoreEvaluator | `rouge` | ✅ Supported | -| MeteorScoreEvaluator | `meteor` | ✅ Supported | - -#### Pattern 5: response + context (existing) - -Already implemented via `_EVALUATORS_NEEDING_CONTEXT`. - -| Evaluator | builtin name | Status | -|---|---|---| -| GroundednessEvaluator | `groundedness` | ✅ Supported | -| GroundednessProEvaluator | `groundedness_pro` | ❌ Missing from frozenset | -| RetrievalEvaluator | `retrieval` | ❌ Missing from frozenset | - -**Fix required:** Add `groundedness_pro` and `retrieval` to -`_EVALUATORS_NEEDING_CONTEXT`. - -#### Pattern 6: tool evaluators (existing) - -Already implemented via `_EVALUATORS_NEEDING_TOOL_CALLS`. - -| Evaluator | builtin name | data_mapping | Status | -|---|---|---|---| -| ToolCallAccuracyEvaluator | `tool_call_accuracy` | query, response, tool_calls, tool_definitions | ✅ Supported | -| ToolSelectionEvaluator | `tool_selection` | query, response, tool_calls, tool_definitions | ❌ Missing from frozenset | -| ToolInputAccuracyEvaluator | `tool_input_accuracy` | query, response, tool_definitions | ❌ Missing (needs tool_definitions but not tool_calls) | -| ToolOutputUtilizationEvaluator | `tool_output_utilization` | query, response, tool_definitions | ❌ Missing | -| ToolCallSuccessEvaluator | `tool_call_success` | response, tool_definitions | ❌ Missing | - -**Fix required:** -- Add `tool_selection` to `_EVALUATORS_NEEDING_TOOL_CALLS` -- For `tool_input_accuracy` and `tool_output_utilization`: need - `tool_definitions` but NOT `tool_calls` — need a new set - `_EVALUATORS_NEEDING_TOOL_DEFINITIONS_ONLY` -- For `tool_call_success`: needs `response` + `tool_definitions` only - -#### Pattern 7: Special — Graders - -Azure OpenAI graders use `type: "azure_openai_grader"` instead of -`type: "azure_ai_evaluator"`. These are a different testing criteria type. - -| Evaluator | Status | -|---|---| -| AzureOpenAILabelGrader | ❌ Not supported — different type | -| AzureOpenAIStringCheckGrader | ❌ Not supported — different type | -| AzureOpenAITextSimilarityGrader | ❌ Not supported — different type | -| AzureOpenAIGrader | ❌ Not supported — different type | - -**Out of scope for now.** Graders require a fundamentally different config -model (rubric templates, scoring criteria). Can be tracked separately. - -#### Pattern 8: Special — Red team - -Red team evaluators use a different data source type -(`azure_ai_red_team`) with attack strategies and taxonomy generation. - -| Evaluator | Status | -|---|---| -| ProhibitedActionsEvaluator | ❌ Different flow | -| SensitiveDataLeakageEvaluator | ❌ Different flow | - -**Out of scope for now.** Red team requires a separate execution flow. - -### Summary: What Needs to Change in `foundry_backend.py` - -| Change | Affected evaluators | Effort | -|---|---|---| -| Add to `_EVALUATORS_NEEDING_GROUND_TRUTH` | `response_completeness` | 1 line | -| Add to `_EVALUATORS_NEEDING_CONTEXT` | `groundedness_pro`, `retrieval` | 1 line | -| Add to `_EVALUATORS_NEEDING_TOOL_CALLS` | `tool_selection` | 1 line | -| New set: `_EVALUATORS_NEEDING_TOOL_DEFS_ONLY` | `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` | ~10 lines | -| New set: `_EVALUATORS_NEEDING_OUTPUT_ITEMS` | `task_adherence` | ~5 lines | -| Document that default path works | `coherence`, `fluency`, `relevance`, `intent_resolution`, `task_completion`, all safety evaluators | 0 lines (docs only) | - -### Data Model Gap: item_schema - -The current code builds `item_schema` with only two string fields: - -```python -item_schema = { - "type": "object", - "properties": { - input_field: {"type": "string"}, - expected_field: {"type": "string"}, - }, - "required": [input_field, expected_field], -} -``` - -For tool evaluators to work, the schema must also declare `tool_definitions` -(and `tool_calls` if present in the dataset). The schema needs to be -dynamically built based on which evaluators are enabled. - -**Fix required:** When any evaluator in `_EVALUATORS_NEEDING_TOOL_CALLS` or -`_EVALUATORS_NEEDING_TOOL_DEFS_ONLY` is enabled, add `tool_definitions` to -`item_schema.properties`. Similarly, add `context_field` when context -evaluators are used. - -### Data Model Gap: DatasetFormat - -`DatasetFormat` currently has `input_field`, `expected_field`, and -`context_field`. It does NOT have: -- `tool_definitions_field` — needed for tool evaluators -- `tool_calls_field` — needed for `tool_call_accuracy`, `tool_selection` - -**Fix required:** Add optional fields to `DatasetFormat` model: - -```python -class DatasetFormat(BaseModel): - type: str - input_field: str - expected_field: str - context_field: Optional[str] = None - tool_definitions_field: Optional[str] = None # NEW - tool_calls_field: Optional[str] = None # NEW -``` - -### Revised Evaluator Support Count - -After the fixes above: - -| Category | Before | After | -|---|---|---| -| Works correctly today | 8 (NLP + similarity + groundedness + tool_call_accuracy) | 8 | -| Accidentally works (default path) | 0 recognized | 14 newly recognized | -| Fixed by adding to frozensets | 0 | 5 (response_completeness, groundedness_pro, retrieval, tool_selection, task_adherence) | -| Fixed by new sets | 0 | 3 (tool_input_accuracy, tool_output_utilization, tool_call_success) | -| **Total supported** | **8** | **30** | -| Remaining unsupported | | 5 (4 graders + documentation_retrieval) | - ---- - -## Track 2: Evaluation Patterns from Real Scenarios (Harpreet) - -### Pattern A: Cloud Agent Evaluation with Inline Data - -**Source:** `agenteval.py` - -**Flow:** -1. Connect to Foundry project via `AIProjectClient` -2. Get OpenAI client via `project_client.get_openai_client()` -3. Define `data_source_config` with `type: custom` and item_schema -4. Define `testing_criteria` — array of `azure_ai_evaluator` entries -5. Call `client.evals.create()` with testing_criteria -6. Call `client.evals.runs.create()` with inline JSONL data -7. Poll `client.evals.runs.retrieve()` until completed/failed -8. Retrieve output items via `client.evals.runs.output_items.list()` - -**Data format used:** - -```python -data_source_config = { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"anyOf": [{"type": "string"}, {"type": "array"}]}, - "tool_definitions": {"anyOf": [{"type": "object"}, {"type": "array"}]}, - "tool_calls": {"anyOf": [{"type": "object"}, {"type": "array"}]}, - "response": {"anyOf": [{"type": "string"}, {"type": "array"}]}, - }, - "required": ["query", "response", "tool_definitions"], - }, - "include_sample_schema": True, -} -``` - -**Key observation:** The field types use `anyOf` with string OR array. This -allows both simple string queries AND structured conversation-format arrays. -AgentOps hardcodes `{"type": "string"}` — this works for simple eval but -blocks conversation-format data. - -**Evaluators used (9 total):** - -| # | Name | Category | data_mapping | -|---|---|---|---| -| 1 | task_completion | System | query, response, tool_definitions | -| 2 | task_adherence | System | query, response, tool_definitions | -| 3 | intent_resolution | System | query, response, tool_definitions | -| 4 | groundedness | RAG | query, tool_definitions, response | -| 5 | relevance | RAG | query, response | -| 6 | tool_call_accuracy | Process | query, tool_definitions, tool_calls, response | -| 7 | tool_selection | Process | query, response, tool_calls, tool_definitions | -| 8 | tool_input_accuracy | Process | query, response, tool_definitions | -| 9 | tool_output_utilization | Process | query, response, tool_definitions | - -**AgentOps compatibility after Track 1 fixes:** 9/9 evaluators would be -supported. The remaining gap is the `item_schema` format — Harpreet uses -`anyOf` types while AgentOps hardcodes `string`. - -### Pattern B: Red Team Safety Evaluation - -**Source:** `redteam.py` - -**Flow:** -1. Connect to Foundry project client -2. Create an agent version via `project_client.agents.create_version()` -3. Define safety testing criteria (7 evaluators) -4. Create evaluation taxonomy via `project_client.evaluation_taxonomies.create()` -5. Create eval run with `data_source.type: azure_ai_red_team` -6. Uses generated adversarial inputs with attack strategies `["Flip", "Base64"]` -7. Poll until completion, save results to JSON - -**Data source:** `azure_ai_red_team` — fundamentally different from the -`custom`/`completions`/`azure_ai_target_completions` data sources that -AgentOps supports. - -**Safety evaluators used (7 total):** - -| # | Name | builtin name | -|---|---|---| -| 1 | Prohibited Actions | `builtin.prohibited_actions` | -| 2 | Task Adherence | `builtin.task_adherence` | -| 3 | Sensitive Data Leakage | `builtin.sensitive_data_leakage` | -| 4 | Self Harm | `builtin.self_harm` | -| 5 | Violence | `builtin.violence` | -| 6 | Sexual | `builtin.sexual` | -| 7 | Hate Unfairness | `builtin.hate_unfairness` | - -**Key observations:** -- Safety evaluators like `violence`, `self_harm`, `sexual`, `hate_unfairness` - CAN be used in normal cloud evaluation (Pattern A) with `query + response` - data mapping — they don't REQUIRE the red team data source. -- `prohibited_actions` and `sensitive_data_leakage` are red-team-specific. -- `task_adherence` is reused across both patterns. - -**AgentOps compatibility:** The safety evaluators (items 4-7) would work in -normal eval after Track 1 (they use the default `query + response` pattern). -The red-team flow itself (attack strategies, taxonomy generation) is a -separate feature. - -### Pattern C: Agent Smoke Test - -**Source:** `exagent.py` - -**Flow:** -1. Connect to Foundry project client -2. Get existing agent by name via `project_client.agents.get()` -3. Get OpenAI client via `project_client.get_openai_client()` -4. Send a query via `openai_client.responses.create()` with agent reference -5. Handle MCP approval requests (auto-approve) -6. Poll for response completion -7. Display response text and citations - -**AgentOps compatibility:** Not relevant to evaluation. This is a -pre-evaluation health check. Users can add this as a custom pipeline step -before `agentops eval run`. No tool change needed. - -### Pattern D: Data Format — Conversation vs. String - -**The critical data model difference:** - -Harpreet's `agenteval.py` provides data in **conversation format**: - -```python -query = [ - {"role": "system", "content": "You are a weather report agent."}, - {"role": "user", "content": [{"type": "text", "text": "Can you send me..."}]}, -] - -response = [ - {"role": "assistant", "content": [{"type": "tool_call", "name": "fetch_weather", ...}]}, - {"role": "tool", "content": [{"type": "tool_result", ...}]}, - {"role": "assistant", "content": [{"type": "text", "text": "I have successfully..."}]}, -] - -tool_definitions = [ - {"name": "fetch_weather", "description": "...", "parameters": {...}}, - {"name": "send_email", "description": "...", "parameters": {...}}, -] -``` - -AgentOps datasets use **simple string format**: - -```jsonl -{"input": "What is the weather?", "expected": "Sunny, 25°C"} -``` - -**When does this matter?** - -- **For model-direct evaluation:** Simple strings work fine. The model receives - the query and generates a response — evaluators compare output_text. -- **For agent evaluation with tool calls:** The conversation format is needed - when evaluating tool-using agents on pre-computed responses. But when using - `azure_ai_target_completions` with a live agent target, the agent generates - structured responses at runtime — so simple string queries work. -- **For dataset (offline) evaluation:** If users want to evaluate - pre-computed agent conversations (not calling the agent at runtime), - they need conversation-format JSONL rows. - -**Impact on AgentOps:** - -The current `item_schema` hardcodes `{"type": "string"}`. This blocks: -1. Dataset evaluation with pre-computed structured responses -2. Tool evaluators that need `tool_definitions` in the dataset rows - -It does NOT block: -1. Live agent evaluation (agent generates structured output at runtime) -2. Live model evaluation (model generates text at runtime) - -**Fix:** Make `item_schema.properties` type flexible — use `anyOf` when the -evaluator requires structured data, or infer from JSONL row content. - ---- - -## Synthesis: Combined Gap Map - -| # | Gap | Track | Severity | Fix | -|---|---|---|---|---| -| 1 | 14 evaluators work but aren't documented | Track 1 | Low | Document and add tests | -| 2 | `response_completeness` missing from ground_truth set | Track 1 | Low | 1 line | -| 3 | `groundedness_pro`, `retrieval` missing from context set | Track 1 | Low | 1 line | -| 4 | `tool_selection` missing from tool_calls set | Track 1 | Low | 1 line | -| 5 | `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` need new set | Track 1 | Medium | ~10 lines | -| 6 | `task_adherence` needs `{{sample.output_items}}` response mapping | Track 1 | Medium | ~5 lines | -| 7 | `item_schema` hardcodes `{"type": "string"}` | Track 1+2 | High | Dynamic schema building | -| 8 | `DatasetFormat` lacks `tool_definitions_field` | Track 1+2 | High | Model change + wire through | -| 9 | `item_schema` doesn't include context_field | Track 1 | Medium | Dynamic schema building | -| 10 | Red team flow not supported | Track 2 | Future | Separate feature | -| 11 | Graders not supported | Track 1 | Future | Different testing_criteria type | - -### Recommended Implementation Order - -**Phase 1 — Quick wins (unblock 14 more evaluators):** -- Add evaluators to existing frozensets (#2, #3, #4) -- Create new frozensets (#5, #6) -- Update `_cloud_evaluator_data_mapping` for new patterns -- Add unit tests -- Update evaluator reference doc - -**Phase 2 — Schema flexibility (unblock tool evaluators with dataset data):** -- Add `tool_definitions_field` and `tool_calls_field` to `DatasetFormat` -- Build `item_schema` dynamically based on enabled evaluators -- Add `context_field` to `item_schema` when context evaluators are used -- Use `anyOf` types when field content may be structured - -**Phase 3 — Documentation (confirm patterns work end-to-end):** -- Document which evaluators work for each scenario -- Add bundle examples for agent evaluation with tool evaluators -- Document conversation-format dataset rows - -**Phase 4 — Future:** -- Red team data source support -- Azure OpenAI grader support diff --git a/docs/media/agentops-diagrams.vsdx b/docs/media/agentops-diagrams.vsdx new file mode 100644 index 0000000000000000000000000000000000000000..0d4a9d0dd789315d4ce2e4c9457799d0a1483e45 GIT binary patch literal 1958863 zcmeFY1CuC0xFy)OZFk?cZ5y|3+qP}nwr$(CZQFa_o1GW4yKjHNOhrXjWMxK1=85`F zWt{KiUnyV^6aX*)2mk;8LV(6Y!tQZE0Du-K003kF2p~-%TN@{18z)^QcROQ8ZCW>L zD}n+LAo4r_pnv`U&-K4C0$nN7vWN5tqqpLpaPhU3Lo(`W{4$XoQUN1!<}PpQ0aWb9@Y0@Lr45>#Na+!i zEhkL$mhd`JB}B&Z9cOoxa@&Jjz`5qlSi zXa3!b#6G2BT18@7C1Od1#Pl(Cy_3XRcZ}n(Y->X}E<#=-Nd?F_KhQnuv6O;wxU4ZU z!I!7xt)Aohn^Y@MJ1k3SRbAz9$lNhx{GF$&4v+RhDgzL3#v{ zSMjgFWADQv?x4SkHV6u?5W(;mR}uz@d`5B=FSqqjphTp`^Rzs8yPhUBT^hI2vn?0} zE-gnQDUKm3oW<6f53L{W_kHRlbx5}OXiVsd#1?_&>g~#ruf!zJY)&C zb&w!kgH-Z4VL^BB!-vjPm2&^vsFGtDccVcB0PuDL0Koj$!^Pau+?LK--_gm~!SR1S z|Ksm^rEQ(K!H)EkQ}G4vdsqW@=xSwa>|ji9RJJX#Cf(5LsawOX*QY*mLLW4xwe8z` zlVUun$3#!pxelV}w`9}2(LlrVudMyKKQ^KDhD$Y18ajooE@myF$0I!?&&gq1sSLkj zF7oa2c|TY-j;ryL0s3L*v7J(>9EOibh%wohiEFVbm}ito9Is%Jv`I2z3>VFR-~cL* zsIlm9Ko+(SO#aqySL9R=Vi?ONnatEnZb>+1M&AEZ_$-4_%OAF(lCsaO=AUy;tx-gB z9qNP_HSjoMfLXHt;JD_NeNF-U^%$}fw*$tuKl(Ugj3icRv>pd$?wVbbkQr6O5Q7C~ z78}B>MaCAn><&B;lki5_5qRNuHC<>PrBIeJsmGGxY@?2sfuPPuJJKJ);2Tx=TP37) z-fOaH^~yY*tgQUFFQ#lOfqRUIwK|L&;=>a>^Z>&=yo6#ev&{0-RR zDR)HC#q|g^mg;2}XH_PcQHQ#uF;5n3+Z`D)^R|!VwB?i%j*OBEQO#B5#g%D`o82(Y zomtsYE~pcIG7T23K^5iAdKu+LE99EHSUXUy2<5j)J>b^9W#Gtu<$&vH%Bn=oX@)S< zdFQ5%h`?bRd+wy%g!$Xs!C@TQuPDIf+b+-7L^Z_Ir`1Sz<^X7W_}kHbI=fOGu}3rK zPR&0#nHI|DRob)n8!rbGwz~4rM|rGQ;KUujdl~b>AywJCq2+5vq$!o^br8NVwrt}L zFs?k}V>c%yuB|#D3o5i>`O$T=aXTb)wZj51*C46If~kNyFSWY+*O1;Br|!73m+pZz zuS`YykI1$c?%r=6+F$)ysn=J8OxB9Fam7J=NUBGjaYJ-r047L6zuJ;=`I5H_udHMA zPHU@=sK$p>;tH&Ey8ISp{BUJZcHB0b3pe%y+_$kP=y0N37a?RZIjm?tZ%|y2Hv~Ug$8!5v|$w}b6X zUxZk>Y$<^Z&yrBsGE(e=Q}$qf!SZQ>bjqPjl$et2z#c>8O&P%o5#KpULzO`dhO9&q z&}Tauav-8kqb`8b?PMZx{%MHwZImr{;DDvEZ2o(C{h> zpcZyBR|T-5GZNh!FsB2KBNSxZIv_GuFHz{dGFTVSq#9v+7c=uPwYx1l?4*dV+My)i z1YdIlHtitg#BUR$ncq+eeF0bQ$AeLB9wmj7R%A}z?m*_GTB+?vQJV%b{242v`A89R zrU3LIr{u8{ijdZUN%zlTEy>3Psyd~n>+K~*GT$>K$mP!k=wf&ilRg4J>tGCga1?&K z?pfnM?wC7rz=Lq}#{mRPtlb9^z~O&G=_Aal4WnB5{4!ib&4!EjeKUCltGc*Ku*D?F z!%6L9695h(9!RfWa6}|R6F~H1q2(a)_SO6$BrHN1C6)x%fU&|d-6(}WfEQ;8Z|JEd z(lYqj{ZHUS;m@M8h|Cz_Z;AkUJ z@&HQr%aNo$eea7#BU=mNCN1SRChK3wfz1F+zpu;VT9OsXQP7JrAk&Wj zc^FQQeGX@qj;-{oxi1IWYZql1!zny>!zRiYgu=g9aG>M;`Hv&8x#Kmd(5Wz@@w0rAJijPQ2W6x2@?Xy zmIji@BCs^+K->6Xk27ELpK^g?tYw{_<_>%Ho3e<}j!Va;oad+TpJ&8~E#{9HjAoWf zna90?KT*h7{zE!>t*xELFKcWA3ApoR-c9N)-H(@nDkIM>jR?*ljVVD-(Jfv~9#YXD ze^DM)m|^W|)h%Q9f^b-7gVkgEky7=#++wxiXQbVHIWD_eue=@jK2m660gF7v1mT)v zR-t0M-g1F4tYVmlQdR6BwR}sZfFWpUYU2uD!^USS9oceZn((e=Eht)Y1D02a(NjgiL2*>oK5CqzvRC3;tacE)kh6Sz8EvR zh|Wb{oZRT?(Dn7w4`ksn)k1T_(2#ef%fF@1F4@PXDG?x;&6Njq$6$YosV^(IA(&%_ zDh-6+g&SIEc2H$;usUO})ViD60b#GlY7(2dj@N=z;8F0d9Yh~n$(*=JCoxoi%Ekp+Oi7w+PK>v{B-6zvxpMzf$JR?62KoJzw) zr)SK88UCe>ICx>!zV4MUj0xvqTWI9_g3#A?d>~~Ihh4-|wYjb1oYx_vOb}tuQyLHI z6xPNRu=L7FneVhU^m-j3Xq~~HbCyF-c6RO~kMs*Oo1J@;{n!Zomo;~7$p~EU&(>eQzvM;l{}OYtTo zr`Pn%>7O&Z1|GfDPSeV>i~2-vfl!>Dxw}-pV5q=DNY1?hdi}xl{D@2Q*=5y@XBq); z4Kr@zs3`m%H zkM5YNOWfTSnbkROPkixpjhNphCK8KE*wOsvcav2bA!HgH8ZpO6B5OOi4fXbyZ8dgT zMG~>WB&3E-?E}$PSEk~Sw23Im_+-k|c z1w#I9SVI9Sya6IOF>BOt(zp8(KC84#k0o~R9*SyL?3@Kv-}Af&=-d@?ve}!r=VemR z^I8tipz@{tUKuf9T7hWvEc{`h)ph@O0}b{ zn{HwSx{t%X#N&Za&VwMfL0v-}W$l}KE7>=wtBfM-O)?kh+aXmaH&2MtcV!8Y5JV9Q z;PTstFm7a0a!4+VvqTT=yc)i6IC@e73g!kf)hQzM=gaX?d+A)kDZv62afDaU;B7{~ zl$Ipc!WGP~GzFcHf)fJcGKTQ16y7*=_4X&)LIwLUH{cKOP{8HO{JYW`%~;$wQCEqJ zEb1XaEvtvf1WATA+5OO?#1WeLLUhb`J-NTWt4OOJ^wElgr76?~Jp~BL=-eo#ZAl?W z>VaxWZi>nlI|1`_?dGm??F`n>~8J_FNCVBmzikV zP2GB`%zJ1(c^cv8Ahvw7YBpQBUm>gl{r$|%9~Qe%f(GF3#rrl1z(QerYWwitJ1Dww zbl81q!~y?K;1s#^Fk@k1M)?J{{@z3`he0QgK{VxFGjlwbjZ=86v9n$<2g#1zSnWT& zsT9br;%8wvW8u%s>X!r)(&4iVMy6vzaglTM?N$q{`g*#$pU&sdyvTqHkw$KUffq}) z)TA!bl>n4v0WOC+FF>q<@*Sw%vu;O4e#i1_z3TP?M_=;){&jr?Mren7nibXO#d`F` zsi51J5@M~6FGN{6*WJ;^y6X=7PKV<^&YabyR;+MOG#{`S2`oktUt0zwq0d&^PKP5P zTm`rc$e^Q{1N@Q4;j(%0eWaT;eHR2@kxZh~h}&6WD0E)Z@-BNnhPE=D6WnT}C5n#8 z(cmvKn67dWJvK#~#aTPv^BJQW(aynlCaCr79r!WNl~5vpM{ReLrzAik zd&`sbQ=PH7JJ)bi#x~WUG63k2Irv-gs#>w89Uu?0#=SReW()&zOLTADEM+4Qy5`yx zvwP{1)w8A2&Ff~MsqMy)aWht9=qan}YF+6a<$YStQ(uj?rwT+&gd)&J)TWb3!_U;I zh1zEJX>A5Bt)YzJ=pvOb=zBmVA!#NzWWIY%CE=pW+duEH!u_d}9CG{k8P_E!G8pBk^L>9g*?C*i`^L;g&n- zh8ux;K49r`*HczjMm}W8nTKEr{&gJcB+i9W?4}%=R1y6ngWCd-OZY(HYYH8d+atlJ z`|kHgJ*$2vb>0iMq>Z*~#}LE=;p%js~v1mOEvE60W|gR0AD*?aEqWm`8usn)&<&`byz}T0DTfnac%7xz%q%4t6AM`B&0teA$ z-~*j5j-~+}pSh}qlU11piUyo;w)*G7>M!_zr(E)CCdQ`z6IYDO005BwOJgyzHFUQA zm%;cS;)!0jDq!j+$iAFQ38 z>$R&t_ZPjtmv7o2arTV6wCj|W0_DpmML%)Im$`88Y>(hAFl^wHW;tY;N3>jNeY%KvFBji`6sQVYgu)^G_=yrzVfpkxBC^v3pS{!N*o zX5IT+J{who99-qKiY@x*#Ff3ADIzA+lX(=G8!`<&P`7{}usDsKcG#wDSBk8YyrXxM zIUmRBO$IY`rT@aW*Q+}k4QV5UQg>97tkToj$;<59i(50;Qp6h1IR?IZ*Y4v;aYSVe zF;lg=1puS>Ag=#{*CbQOZ0}SV;=(6eG;*A>D|-Dv`S307?fPCh!^nzOqr)8)2Y;jY z?^CGVGY851T5asv^@JBN`IbG9@& ztr*k-2bRnTND=6vjGZ}rj7K6d)Iwa`%DgoSh)h*4p)OQ1yRNkJ1A?1n(*Qsjb-?@}%fsBq+lWaqa_ z@8+!{Lh<%Z5&9Drim-Vrj~VD@}DcPoA(czTryq=LaT_gIBfv#Ca z2PulTV-{>md8}eQK~ifLYz)^RJCdlii?{9Ze--}Ct5oLSyqa9}rNf(V{?hg-u5r)} zkXC(-KIF1|y2PgFwPaGOpMd%)r&gr3(`skin`EHdULzBAsMuIXZdNfUuOWPvoD6fkDz~UXB$%Dha1T zz17M+F@cYaB~z>vCCT7y4GUq4Qep_bs7xByk&rr&{TC}tGN;UU4_=A2p6cEKy5C(+(uktM`O zoB`xrpdZ&ezmwmsXO7Gh2xRa0Q-V6tOu zUTb@1z%&;wr`zVcePco=uK@@$V@_&S zGG381es0mvJAYie&IxuINY(?r7>56-{UGAt{yEFGZvps6Kcb^UvJf@rgW`ZZQYzwM zoJShMf42sm{^rYRYr~$P-FYD0YhvwyHOtp7{EQ?I{YrQ6ZBTjQMh)r*Lr{X=-DctJ zCue-Pk@eUGE+)zvCxZ!_Oi`KKECmhxMbqi)4lS)qj?ECGy3r_8K~RUpbe?#!8%fx- zUk#Y}(bZ*U!LuM59a1{@DCSBsP0y+s!uS%F>l)^2X^|l{samHeJN@9n`q=d4-7tYR zh?@*G)1h6E4{^7H9olj+z?}iEk^WY!cLdNB28_?<0fG~EcchSAq0kWgTAGx)v`>qw&N&W|kvP2qCenZSVc896a!oF_;c4t?zyz2_)E9PkL$hcccXXOtH z-34(ET&fhqN0mNDfCxM2Z`N5i@V-Z!3)gKT>gLhnY{`I^c>i^VYbQgvStU$TvM{zm ziG!jM?tz9#meo)Q_@PQtlM%ic_8J&3Rl)M7fpa^cQ0#Fu)9B1On0Fc2vVm7eJa#8M z16XtZFq22v0chBbg+Cp-Dz!GFQ$12jJS@Q#F}9tEt=npg6n2bN4GniRZ~d@%#ayxO z@c1YaBhDw|e)UT;lE`H2ya=G4SF#ek5_k3YtxJ0eC_TUr6bUrF7bFihHNf67Q;R;9 zm;)x$h~tc3rM{r`VppNbWm*Fg3(cVYo&`Prkm*(YFzs`OwO;I?^>8s{Eif$$t;|iP zosnaBA$2r(=Hjx4)KpZzArej9fd0e){LsiGctI)CL7%QZ8E%}M}A+xk1|kbh7K&m z2)i0|Pk3NSGa+&0WkF;>9||`r$bhpN3jbCFYDcRH&Tscz{j(>Ewb0)CE#wzOm8vN1 zYF6B90f%s~v&LWgeQl|LOtf#LM}gA1?*AbMg@$208wfnRdKN_UY*mw&?5BZNbT znvvuiG`~73kh}&o8%314o`rv8Kj1}0Uq)sY+>17tEX4ExG=)A;VV3_U1+DotX3dt} zNhQF1;Y334D=)jt?rq+0OW=<$*I84LUE2PRhTUXaA>`B=HHdAZ6O)4MS}vyMVLue@ zHN@a!#&*6fa>l8>tR=uUd`8Sb8<7R4x1jfQtboNb%V<22yRZ-^%@T}Y1{w4kHqQ|} zbtd0b-}vbS*CF$*0>M}ysg`m0h(<|3-ob$3F0@8#fF5x_>b1o8#fViEEvE;>{*H>c z%N#=L@;ea)zn54eKN&G^p$W#`Fn5nueR7hZo(q(!0|gXYh5~@AlxNZ3PH1#D^sw7E zyg3jYWqTZJyt)1J{-Yktr{u@x%{wTpP+4H(HehqV*}~jM{#z^^;mJmvm$$oz2N;`K zZyVjF^eN+Z9CMmkpEA#2HBc%V0Y0?gW!Y7~*t5sSgq7&UJL0NA-!ho2(x~Nmx+Hb6 zVxZE>&@#5l)+)jI)>4cScWnRE!$(;_=+yC_ zlskrf9mrJlJzP^QIH6QeF^3)afmnqNM5g+8J#O$SPl_#`z}ffZd*_GvQFJiTF7O&w zEpWTUzC*1j_j3kfZoCM2@pj=_Bo0;B5rgo+N@$%%=YV<7Nq@{Id2aGS>we8PMJdBI z)CXGTH(fz;I+^tTb-wGO! zsbG{IMV8$OFj!RweGZ=aweYL1?66k(^f{grRl?{K>hu11$pXZwQuxJ6w@so<)-KW> zU1RhLchEM=GD|6!m}cabBX6CC9+&|vs*8Li6FUc~aMWm&=5H%j+X$;1nm3h3lpxGt zD}!2%X~OD6kv~yI;|H|ingyY&ghU<+pt{XZRQ)Y6GgBci(1J#-6ZIlun2vA-JfS?$ z0!kYSc-sF|{)8=!BLVz9stCWZq%8i6%-K~`J*VjUA{ zY@Wgk>eeDv-=i2Yh0-ArxoAWse-yQop=a~8bmPJHIazs0W&Eo0VYK?wGuTd5dUa!} zIm=6pDm~a+U;WC~q*=wGhb2@}Zs7l;bSXT;fUD#eS;-q=m=s*U*luSvfMTkmwDJp z6u%~J@g&gS-@;V&!%ubx7cfX~9qQm1GCH_QDg#$z72;@A`*(ao&zxitW~O6aV(~9Q!u6LufnTLzrdiZ5-|~MI3*NfD2U)Ns?E31EWW=uy%RAaB!cQ& zOr{AJV|~?Q8U1jCTe04UV67F;&S?o8w?=zw)a0=%55iHwCE#hpOLD(lNbM9%KB}yq zk4x!cH3)#a_jOoxCR^g;cxV6mYf9|mix39xS!gDkD;M+|4l1)JW?-=>1SUr@an;wN zn8O3H+vK(xtp>feal4shwnxsjl6?AWKnhrB?4PL#UH;LwtE4b^xx+$RF6g8!&;_7m zEyxAXD6-pp4eH}p?>QDs>-9pide}_-%*s6G17N4m6bD*(@8|^Q6;U_CB&#CC*<(b0ng!Y-!f^Zxqi-Y^oZ9ZR@5{k{ z$dMVKnv&qIw2iBZWQT^}N>oS2uve;<(XWN}22bkij=#>%XN;8QuBL@f=@*y96MHl{ zX=ml7d!Ed@l0ZA_5Nr%?uJHy9^b9E?IyX90ndAl}NUA6xt9r!kZm|k>|6E7Is8ah4 z-EtJT9CcMk;u#XM|0;E_itMM_>!t&sd?-EZpwBxOCi0(Hf3Py z&hJpQnpUA^aa3%o;tVp_J&d~zIM_}KZNP!asj@JxaRbTvrTJV0ya>zid6?&w^!Q$G4ec9q8TLE|^seWIhsn$SxaY?{LWrI)3#|6H8sk zl-9GqV@Iu%^($STe0NceqbwJ(cBxkFOYa^C3K%(F*T)`bA@$X%oEJ4K?^?!{>>tem z#9|^|LJ5I#8+7Hr$ex0n5cve|HRUwDyx=b&EqfoEhHrVT?>*%-zkU|ce$UJ!evSE3 z_O^+;BR5pHHga4|Ooe_UHhg!|dy8LkT~BPwzfZH@H7$O}Z>NmYiT6Op2iZ#QFef%Z zdpMO6?vFrGg*b9#HxjSI;NSxCl;4PvrRKDYvPkHfz(0p$`vn@kofVND?zMl+swM;Q z=XAdyY@2=XzaV^l?b_Uk1fb4sdJy7(q(!aBQ>RvAzsbd)j01R~q$dk_pyv3{=w&*} za)C=PXZ;s9fFF&3OXnLe!o4V^@q{npN<~9Em!7f4ZFlYrs@@MqMMpK($xG;QHKIC9>@P4;K>O}CV(Iq@DN_; z8}g7|^xF}K)XEm=`ziY$E5jiO7#w39mdnSIyd)OlA;4iCp3B2SwfGf(to48!@`_ye z^V`JvBrFA{r^_XJ&FbDi+S(nXvAds_wLP`h!^4-~5#pE}KrX@4=fEI_ zsZgpd1ne?E>p8#nDL@7d{&d7wrEC4xs967mS53@~vKiCw}j7Fz$17L0|^5!lXqMq1?HZ1SIVA4Lx}ta+p@TGotq=;sn}jFNp=$ z#UKc&*fCDK(V2(v#kZhLc-@%C6;0Tk|6n*oTT7QdsWs#9s-WTIA;7O8(w?PLqBi(g zWm35$a`mW_?i?H77*?jxcu_>m zC}sq?*3NZr|0@w-jk!;RA1}gNc-Q9E4TQ~aoyp21vGNEBH8_SGObLWoz;i_ zlTZE+IJ0}D7;Ka&#~Bp-yeIEpNOWdmcN~M4ETMXanXBVj4`u*~lNw9JV|`}6FHD@{ z`Z@MiOeOD{$A2=M*2LG8nYYg0#==U)=|$|yWtdYZQ~79_-DC;td*<<_EYgPh%}!G7 z%AvVz!kAJHNiz8Dvdu_xEc6u7r56wLp|vE!qwctQnCZedLV^tW`$9fY>yrm8)gUG9 z5Dow%OImC`D;?a}#CJ44rit8xq#l;Fzk9DfqgW!pg^#Zenboa#H#LYlddv{FWwi<~ z81=C?XPDGzcR<^r*BhkWfk3=i%&_ZdH>1azC!)s)1L;AV(mg^|J1wjP9CXhxth{B} zQbPTYC0`A}w$&Bl+fYdE|F2QHu<jCqONG_QY{xmDIW-H-W#@p_ z#=H+dsW**3_~w{csdM2zT8Ax)jt68f5l-m$Q5_rd{VGBIwn9jqow~$uykV8kv9V!MKx;{#X?Gx5H z|8ySz4_N>2+JB_T{u9-v<^F%D!hr30>fa(b>EaYyZ^UqR!+PLu1E&qi3?Ol>l~mdC z^aT)?etBJ9qInF{0{_|RAd|vL=n16 zDSrjnviWtzfl`lY`pEJpbV1kFr>&_q%qSP47_sWQ%fDWqq7Y#Govn5qaL+LM;k50}j0 zNXt7(Wk74)$vP?MaC3~AK~06?7w@%aT_pw6x7TvQi7j;ig?6GM6UI6whb^S+#}}O; zrHR6^Ie?&u7$!JNm8;ihE+UHJ){_0OhQO|%af?~HrLCq8Q80FJHm!QNRRt zFVp#YNnL6$#RPER3+SVORyRxNX90_xWiY3XD+`95$ePGsY@yxaKU={?f3*T9K5O0F zmLmh_i*)O8H_RE;gVFSz9I-r17f3dft^2w__U*9N-=@EY*=Pv0NjX(uh>bXiueJ#=;-wfMrkcCUwd zFZ`O@+ZoK$^4EZrP6|6aFBQioot5X3##-y(7sav;{%HJt_qbAAzNDXQDR&~x zg<4J$YN6CX7p0>JO7RV73==#%VH(_$D>({4YQYBgG#|yc+?p$`Xm0i zGGzV6)RQvOI~FCisV%q+7xgX&dlj(zKzh}&125OKi?_tGpx9_B2dWXUcbFbd84n5t48So`5IcqZET#oiuFFf~>vFTX4XS$sow&3zrT0UMp}+PI;F_4BJyhX2P^ zq0GCJV>&+AmA!cvS}JSa*dX#;0bS&G_(m!3;{gA^C6Cy_^sBaiEztkSnBf0K9{+=e z|BFKYlZ3TqXQ2lEk?>vT9#_(qHXA%qgA1F!>O45P_Ep3seg@IU%Sh%UhMtl69N4O^ZOafR?X=)B8W2bpT?pvs2y?ov+I)HJp)So`70(!6(Wi>%jrK9 z<1-jAEEI-E(>%#~l(EC(R#$`UNuMn|V-J>oI_U;kC`tAg{93!|iN9BS))GlK&Gi&x zrKdnqF4tJc+@aKI2X&Nhvf6`m{!86j%5(qys$DovY6f8e08YgJ{|~+}{HHfV%i0#3 z4e7_H?;9NdL13mo{{2T`h(b$K0*2j&R%LUmR4{(E#jar`?x?9-*B9P3weyJ9csyo9 zY7rwWP+ZGlEZzMi9n&Vq_w#bv>C})_SNdUHAofMV_D^Q?W90bNMV@PGmzRkL*KW_- z=Wb3fCQZuv7kMsCzSXY@bNcgyX>Yn~&Wvp%=1KR|DO=%jLGEmA1x3u`{WfgMFp2Q>Y~#5Z zq&@m-;-(10CR^m@JUiFf_DrJ)>u=h#4mV#f-nL2P#cg)tVPMYHHH>fPaGC1k*G)^S zBWJqt>Ev$7+t7=Pj%>-|iSFRls_}O%`fO~&xzp)Y-m43v=kDhFectHDfZcc!qO?NA z+g|$hwbORTwm_m|=5r?Da+5VM9M)G7ZQ&*O(%_L3Y%jj~`{^)JP@KNu2<`)se;@#?T zh(apyckj%s;X7v-UVIqco1Tf3+lI;Q)x^k)Ivu>69QXp$ zZ`a{MX6}(ktBmMW^X6RY8PmeeeKVhA<&~=(?ol_ly2tXu%Xu$mu%v|&jzyuRUJNlhzG#g5ZJU+vX9fXp ztwgMxf~TQ<(?; zArM%+SQA9o7@-(JB{O}YlLc0)DF~;WQYc*BM4~FD!-YA!u3)@cXxVe|5cI8$!W7W0 zDMkj9rv^xz375wDTW0ic*B4_CH8D#b>S6Qvv=P=;@?xy|bRNL!VXWwz%kqHYuL$e- zaWjJS|Dt4qEi!gAF*q>4uydHP*PQ%%(<`8edu<@L@LXF0As)t`(f9THwKQK;NDx9~ zmw2F(x_3O~an}QP#Vdtt>#|@({brBj`TgDbN9w_`2%>!U*EzK7#ErMZqNhsjLN8*- z$zKHLCJ5sVA;N>sn~QptdJGZn?v0rsP#)*K2J6hk60F4VUEaYefeh!*D!h=Jruipm zFr*aT$E=7=8EZ&>l>un6MqqmTRg*|@6j=k_uwY_LI{dOE!_4=sS&ar;S`_1@Z)jq( z86rclv?NMpYPyVqrM=5+7z55T;csVLfesJla+LvrW@RtCuE=-rA&~Y^fUR zRgb$U(R2S&Ym}I%JQ^596$)$yQ1}?ZfoK{=IzehS5{;^W{)7d61-G}^z{SPH*3-b; zc}xtvn+jebJGKO5nPz2AC-`otKU{VtU-LJ}^1sOV7Y&g0svU!icTe128k&Sk`&SK4 zxgO{J^YdtEJC8*oO|OSu6_jE7L~kBz2Q_**I~#d#-B?b7MrCOT1m)R#J%!cin-SM7nJkAPKTl%)HJsXd%v^ z>;aF5f(1t=9a~Ex41lTz5TYxFh`n6|a?;CM(zAgD1XnW2k1tbcV-(b*uoEDvoS99f z=0AxAU&%dPmZW0!RsD(FCfWN{2h@qR*YA2hX@x8b-eL(PvVrFlww4rJYo7 z&MZ17o2oVF;(;=_`2aj0rwj}8GBsbWT_smP)85!*k>j?{l#PEF;SaB^lJGl7o|35Q za2#te>)}20dUU>^ovM&L`A8cB_PUE3+_L&$T>KigkqzzXz(P3aU~HFlClM8JY+Z_D zq=Pa0$R=wt5ED}jUX1Vvn#?z78r!apyNMxe$=obzds>s0gr)t4eE)Q2)PdPlyEIS} zazy0Pv9g80p#Xq$~E5eF(uXMi-W)q%az}O7-DaqlPv7VAA@!X7891 z@>nuK=g#XgrYx`onjDA^IlH@GnS#`x@F5X7`t73&M z=0PItf~PE5r}puHbSHV>L*H`qeGL0z@Kj%W72vy7507=hg9>wp3+UFssD&fgc-nc% z(Mek}x~%on;)ObwCUc0~{sn>73Uj8b3V`IXFX~Yt zJ_Y4P9YUXLA_gmk;4|X!O$GLm{G^Z%NTNc`Yk>YJ!#?V0e`Jav?lc>Mqyu|}3!?Mq z-k$MBif#>`63C#PB^0)VBSW|$2xUAw4dTfT7vL{M+F!!yqOqG0kIEW7WkS+OY27$^ z8y{7>2Oj^xzoVI;w}4vQpAc_UC2V_*wV~nB>j(p>`lj2pr-=*kNpKkv01hG~YoMPg z!WuuT#)R!m0$f;QoZ&ay|NA)66MKrUTICFWCRlfL(i@q-r#fg~5=zRT0zfDX0R@xT zI`8kuPW9XnBJ6kT?g)IOBiA_>ZPv~kM$;F0-0MyQMxba3!lecHWntMAUcF+_M^x7} zAB=T$&vW%TsjK(PZkOcu6;D^Tw5;8+LhIvyv7B|Lwbj==q^`sym>|H7k4FZIqBIM` z31#Civ9niJ>0fh`0@QyXS#4q2Bp~!NTh_w-egn`gN_RyDE9m4VwqHs*+*7e^ph51R zK`%ciL47BJ&lwU1v#`s#P>M|l6c)X0TcN0r@P7r{}_-Tbu7l>rMs)LN1Y zw#=-r9x(kM^3$|&+7Ltd z*~$0`i{h@S4+;s6VQg@SOsLPm5F~vk&0}ih5Qv$pSQwApJomf3kn4vukwJ31ko*A_E)- zj6tMP$fRNeDv4Mp^((-;d^VXIM;E5PSL=MUU-Q7*#jf|4BSR8P|9539faW zueiIdTou;!xxBczEv(DAX2XKzWm#5h6(e?Php;=+8y8*5au;tLGR-z|gc@(~DC*yq$dmgMv?RaS4r=@%fWLB+MMyVSpWC9zPJyhl$L2!rT zbqG0~zJw)JQ(75TS*h|0SGkNzM;6<|-|!t_HVGnzK9rNElJJVH+M=axEp;^XQeQO2 zdcWt*$=^k!j@@F})4CSho*kv9w+l7GcQT5w?&?i{w4t6F#;ke3z^@kv1I=;n=FfM% zV)vQ*$%ve6*~UKytKHX7*Cl45ZQ{DmvsWHg*2bZ=y~_C_oE4lqSz6ouHrU_oLEn{h zUX{1EmhUOpVuQpd;>0}Nr!c-PRv}-oGj*zvB|;dHZZ`(e0uW$r7|qm%F+p_rbA4OE zU(Z{90P+Y={lps^y@jK3VZ3?2efhZT;vZcH2&n&aOyK{F{Q)v+aH9jQQZ@s1r*jB{ z36{2McLUs%LpO*q!u}fk9t+)0^Bxqk&7#-njv^Qza|Rqtk+fAtY=fjsXrRmK%jD#b zvJWPR8$KFw8;*b}VuI!99zKXZL;HfsfT0dJObH)G|E8*Am=NAO2yY$3!6$B}i*pTT zLDxb1@Jx;4K|Di#dX*`R8~%iUA3yH@^m0}}mu;}MjDZQ~N>8iEWPk@QEN*`Sb!Jy9 zG8YzgxF(T4NKf>^Y{6_C$MiGoiQ4vAtmQ;_GF;i>hcEEu?dsy#I-5U?v!GXv#x2?Z z)u)g^)1lQm{#Cd+FWY37y5Tcu>jJyoJma#>_ADn!Yp40Tv*ZTU-SQoYimI-gZeyW) zNHgt!TvMY1IW_?TTPP44pQZ5h zzA;phecpf~h+i#k++SH#L5D@yzrnt0a9ijV`gl{+n(JV0YogyPMhQHus%pIhv*4=r zI`H6-)P}xzUk5US#Co?L9BA;99T1=i9v00kHFn9ZFx$O1S-0^nJ6LDt27h5ko4Yzw z;%=@XUD+usSm<5zb-X%Xm>QU`z<;X!y}Bi|TNe+L%iOO({~$^V?dK04TaWq?K-$Rj z{GR7*%&gg$c8lmK%jA+JC*}j&K$7{un%1gm^UBrEG&4-!JfP_eM@_+@gd-&@5>6UZ z0B8-Yt4-ORyOueJRz!%Cx^iyN)jh$s>PM-3!6lPVj)+KlWJ73c0EIF%1%nup*?Fc^ zG@F`}(zQ|Eq{FAIYdRu-IUk^@8xTq842Lje00%PkT6-8${jtQDT~B?*ujS>E?2;F4 zJw`dhlZcPWFbq)G>szU3PO@PXbs1K~+q@65?wPGwVWotEy|ve4D=q^tXS zEv1ScpF@Uq51#qkn6p!PYFzFGj0-xD3rbrd+le=qytdDVB3tV*AH8dNm9Cm0kSwWz z9Co69a((e;cIb@xRYoi;K#zqkQoB`alwdVv$8Io^B8U`B)`01 zCK7(72Fmt0E5s>fI{#ORMS^f-(s80uWmraWAwjRZjF=fedRazX(7M5 ziw|au4>n8Y-Og5u?Q8MKhrjhCz_pt{}tJ`8IEW+$Y$*^AGQCxN1YPD zCh^p*AlZSL-Y({O?ZF^-7n;D!RlhSi@G5^e1AdGwI2SX$`6!rJ?TuwuJA?@7cMCl1 zkWOTZEu)O0dqHzrLrXXRlY#+C2k4NLflnH}c}e2PyQEZvK}z8(a?#cHG8Lnc_@nx< zovNjbdddkz^&+WS`-QN6#OcrOqI&)=SnugLN8ae!3Hkj?gK&HwrfGw%+GLTO*<)Md zykVW2w$Wl)qhhx@ds(`EWf}UTzt$TRDA1sF(SpBIH9;%d>akL8v24~-Yw;}ht)gnH zbzzXNMsffTj=cMB%w)Wz)v?aOYWd+96y365RlyD*(Gf*?Z}tloY8*<^A%)|>x$EiU!<`KljxTJo3H=pbR9Dj{_ll(dcnEvP5r&NOAw4si=J)e%&{KXR z_IxG7zkI1WXs#kZ+h5;7sp!MpndsmcaEFudrb)khc|fUtQKSaxxmNIua`pUdHL-%N zT2sralu%j7kg;|75%?Nb~@kD{`av%>J!p{eEU_3|=T*jB@vRNVaw`9Mwfra9 z{FgE$%w(9UZ$%r}5<3{F!{Pj|R)pr3cW$QixJ8#Sn07DvE9Yvh7-o-&Ar=xl_658G{n zgyqN0q^$I25aw=2e1-m@3xho>?}CbzW6hgC_dPMj573@wRzt6oDB9?|FT$RzzGGdT zTZLnmpVTnUp z^_7k;fs_S}0yo@yoj%0*!Nj zxQW%gUc64sS#VIfgeNUtX`M{;GfSD7yg+;0P59p*`(fwj@8hb)9XG@3u5EWa^V|9l z<`YmzU&X>pRCZ(VtPe&&4iK+fO&L!^fIy6E7D};p_Mda(kQ#XQbbnO9eD%kcE`@MT5-0y znU8Rllw7`tLL6=Uu33d{+%dp30o!^WbXFH#C!AQG2ct0PiT3mVbs_ccqS1?)AA`OV zxo(ymPrF5CEY?f{zP$w@;Jg@24ZCjhldqf~Ih1lNh!bPUtkY>A;JQ?}6(~TcDN~F- zr=a}R>|0@J%f3xD?%+7;-^LEynN3p_Mje_MDIZeSzV}6+-L$Ov(va z53a=STCAb5Z~#oo0Q41sJ2^#+if(@I>q;0Ax{hcqd+qYOv;P}vPrVAzL1CGpq08H(U3OYu#Wj#jb=+$WkA}Uqp{#RsGI0@m6%SX=TUQ# z13N!ey$XIvX&>0XkPF1ZIuHfVl*6liv$Sk^4Q}E22}*&tcU|ZPu|p?$_j_~g7o)DI zzh%Zjf3z7GVRr1XQMh}pY)AL-&vax7&9sDyG^i_Su$7pp8;2m8%p|uWOH|mE}xp&cOi581 zc%kt-cbCsuVDs#5r$P1gdwuQe!Ob!>0q5O~vZjZEo|}f|E>T6X@vTx5wlb}oR?y>C z+Y9?KCei-syx`~KO4apo6>wKoR;lS^!>tqf_(8jReqnC~cf|iTun{Y`0+(+H@3pg8 zWuSpj%v|nworYL~Ri7anTJVKtFEw?O%Ib%^-dm^Z@#@mZz~+wrMGO6wt5BrE*kzl$ zAiHbJQZhVpeaLrb(l(?Qe{gFZTLuYZSiz5=x1j$gcO<%%|xCD7l&+Sl<^{%VRl;3(LtQ-hXoMg}4Q5YH0$i>n~RZ~_5+t^`9cdA4H5vF>C|>R%`5EHFE9kdd|Jv20Y?Et==CE3+ZHhp`~hsd*R+cDADa_e@?o zC3<=8@B8`ILI(Lot0Obc&|X$x@GM-bvG<29UuQ#(@N{T@m9Ol_Bka}RzdVBThGqnx z^b+(a?|cF}A>~}Jefjd2cbJ0~@ZO)}@HlJEEMME1_1kw8QS>t#th}F2`#R| zR;=bMCcyi5T6wc&kB)X+jGcoKeEKlX+v%69$Mg1v#rRZ-RD{l9Hi0&V;KLKxCR*>V z?HIss@Z``Hr0gTcwoQ1<@HqexnFo;BM)p(=2k-``UGA^Y*MZY_zFfFIbzdEicY9GW zwY>@)5l=)df=t2H`$H7)k8!tyHz{{6kk@wy~B zD0E~dl2~hGfiEGz?+Ur^G-|E5E0%;iEsLD;?TTp1@}55FCT#EQ5Ed=snVOku=a_G*((-Aj-EJ@tvnmQ?D(PIz>fy< z-<5NtI)!e-4|>oAH8S#dLR?<1ckkpofJK2qm2jkN8>efM1}Mm*@#0AAJ147BWRK`SVr@pcGT0<OlnlhoP-JJ9LB^00zI4od>8Gbf;&P1lg%(; z*$we1z=cMk3Zd+Kv%+5vM}|M|nE=du!z&xs$$94!Us2l&k#v?h!mu>bz+)ZB_8ON)$G<^46PvgVDzvCMK$K)KqU&aK-dqvApKMXgzWbFO%YC4zdi62j!LCqd7yM zWM#6q4vV_J#gUo_t&!ksC`!YRP+#FhhmP1KE43{sB`)igO-Eu{asJGxo?#1$iz@J+ z%c2Jjj>3V;tG1V!Lp<_{dG6F>F*#hJ+1-6FEmF#%I{0@m|AgLyuYLOe}%k=B0$|?76>MBUg&t@k{J@qo> z!bW%TC4o$rnPVttMAy&}KAd5|j3*|;b3~M?w#~_EU7CxTHLg7|@El%Nr@IviL z#C*#+VnEmz^t%D?Bl zO(y7#1f=-C?X8Jvi0R;D;I2ttuJ`tAi+&eVlYM+zfVx)8Xy>P&=RI3FHitU;-PD_g zg`SR++Re=6q__?zV&4`F&;|1iY)tsYL=_7QB!^#pQQocg7c*6chy@6lRWkp=(uGZF zqTq`pYP@ntgj4X5GNaybu*A49xwMnsj|KR|gm|M}fn#$bR zlK@2zJHO9whfW$Xn+U@+Q)8rR_D|>*>L;Bme`25xn!)OHtKNvs!q$%#>O9FloP)kT z(`$#b(2)%!Q>;`bPGI~25#e)qT+*P>xZRRAlSQ?%f7M^ziw8<}G+YRCzrp$g$q`1V z6|&@wK%A}MNU3OLlzEP z6GYVeNGZdD;n!^Q24Z`(oRnyrDTwsiD-L*l;Q8}0od0AwNTkarmJ|F|_6DZ`IKk%` zdh2yx3Sy-zBSwNtrc}(EJDONo{16G$+;?Uh17AoJG{Q-jg2{{$^08!T$rJlbp_7<$ zGmJM$TWlLtirmp3=@M+k5Q}&5`SAPAXgCpQ+IH&vx|rlD*+~2Dmb_PfM%eiRGGpLk zWpJ8qPghp9iq$f$MXg6w-IDbih=TNNBkUf~ng6_K-oAB?U$D{Kc^{WwGsKOJ)O{P~ z7=)=iLy`hUabj&$7$69o{yZU1`VCzJRIPUB5;uj4RIhfN_rQAVq6eT3j!w%JC`SO5 z=}b^H)?h(U+&84aNhpD=I|#v=q`a$mM+9Bio7Mp#XANL9A31D{)Ao3XK;@HJFnmOZ ztP?+~rG8d1iePpZDJe9-t#M!N4kzg zLf@V5`w#k8)Whx1f2C+mx8Yyy5dKM8t^POEV*Nj;rK|l9YN7p~-I#N!nZKry_ka$W ztURKrDH<*(N|~@rAM?r>2?l&t3f$ zwf%K_*U}yfbW%YYwCL%-dYM^et`4_+x$IFPyZm2|__tvnGpyOQJq|H3UDN_?)3v%f z9XBs+mM(1?m{vaI)zcxyw^!oo8f%x8-*4u0(HBjSLsg9w~mud!d{zF zdRbj}enYbKy2{a@AMLcwU9Q(YZ5kLD*hi+m%?8+|8thHA)ZQ#?yW0oK;BUJ^O;N?J z4~Dk3P@&r($V)0#iaUQ7DbzC5UiGLIkt)<&I+w|v&TH+NRm~$RsU&43kxWbICWrry zx;k(aYrn!iS{xsoM9wK;6|R}HSr!(vS=I|CoB(?vPN|}4Rc{sdzHNB~# zsCY6gZ0ZYF?Wl;7VkF(jnz^j?Ga?m_CN3dN?j}#@U&@B!cho)=vle-&_gE3CtWOUV(_+ewrhKAthQ5UX} z6xZ?pJ07I>K*B0sVEq@o>RMB}UiT<5C(J>Xla1}K#p&{)^S!nHfPtYheDNGTJD!iF z482*YKigH*6|QQWRdg$pZ1vp~Iiiv*G){xLijY3(lvVk0$X=!}SL6Cfjqq zY@O7LT3lcm48p*06KQui;&_T-Y!B9!U7gY=*d8a46#nE7X-Tthlu|2P!2f zP`L7m>_>G67KNk(avj=*&}ie|zsvv;OxldI0MkO(MwNtv;9OQ2_4NqB#kpm*hG67@ulx(lKOt2C-r$mfV{SI21mZ=b+v=omA>*KdGNVD2K*z5YrR%Bs}~; z76D*_D!x&Dv~F`7#LoUwdN<-Ot_?8j&+c9j$lp;AL>lI?)2-2lI?XqUv-!vH8}V{v z$)8)u;du3sCW83rhBTxD>i+4oe}zL--P->uXjHfbMaB&j0Tzy82dbG_Gk(}O5`GY2 zcN+hv(#|~?yBW-Ay6ZTc*81zN6FVAYya)G(r?Kp0FlajLNabKCcTYYGX4Qft_nm+^XB;SjymWHlZ$O!S5Vp>R(#88` zmiCP4KR&Wq@-0OB^1-T=|1?#V8Mbr~LNsQV_h3Y*7C3nKdwjs93i=K$MtZV7WE@+f2IlyBRMRuEe6&7-^9k+!DYYUiBXL83?JQoI8m;-X zjMB4)HhObvL$-dBH8?H6&1s8Ta&y3@p?FSRt6a{Mtys5ays0UF)f9zOyS$`XN<_t&*rf>b_K2YNp2ugw8tyU5k9x5BBhk@6;aaDDwJfO~y5-_fqM$3aIAr3O#r}UgL%Rk6yGF+)$wjls z17o|)(ymQItVQOJAwB>7lwe%og0)`aZ8K=&FvvSu4lgU9E%@mO(T*H}v<%chujs-d z(Mj9;#dS|8>r8lmAuftFku4vmHHsN~784G9y&C^{0o)}qUk%$|1^CZeri9A&tK$4*?OCWkTx(hX7UQ!v>WuZ9rTi$} z{TJW(PyFLQaXg!mySg}EnKyRo4`&XJe|vmui;khdzs!%`>&&pGRo^k+?|<#%Kf5uH zgaLmU9}H&PmmLnj5At8!nMI?6zx6c?P7cLAhWRZWPGj)H-T4`(rA3gR!vB`fW-;dC z?EQ|&(Ie~6;QuJru^W4H_W#Cg>yhnztq$k;!MlUHm7>Rk!_p6Xpmz;gi@A^f{fbWu zC+T5yoaaz_2GB#y&uXC{2Y@2VR(mFxXC-`XUv}G}mT4^nKp`ez_a`MR$o7~2+h;Mg z&>wuCa=8`=N#T2gO!#P^ga)B2Lo`ovFZpKZ#?=$PBu3zti+sO#AmCZ%9|%9-R#(l? zNT}ZnUx9#j1iLSU{)J#4qr-`?zJ>WCF-TwS((n7e&XA=$j)Qe*kaYmSt$(^TVF30o2r<(Z9KcM4o9Fb z484VCGQteNh>@|z8yb?;_@Qe~>novf|5;=+!I;?$mlI2ChRw0CKnlCO0YhZ48YZ^E zJde+jrGG5?in)?xi_we={GtAw*VgmwRi(jASltVl{(SU}S8$z*rvIw0L*>I zY<(bBag}mS%AOu7&#P%+8+)Lq!VQ&u8Oi`X5V&cyWet~80lTZ2BLhIjBDHeWeJNV) ziGi*Aj0PJClQ#qB7!}!AK_V#t*0UgF3I!%z`addR`hlD_TtP}E7GCT{@r6}B{!35r zxQ@hu<7?S2@S-|~9r=4aT~HUS=z!D|`V~n^#cavY5Sm zBTpB~@`V71D2~w9HT|9xS=XsT_FzA$vQV7~ORg8>3suF0(5kz^8e5PZzn_CV1(T$w zmrTOg0in}kH|)1^B$ne41}bol@+7p~+u%>dYzP1RAXCPj&xbLdOuBFg-*K`k_)ypi4RZjasX^AV!2m1=5|5vF^(31Tl1F6^mP-NFzO% zXz{cfx=9z)o_3IOcss1zwJia%&cffsZ1qv zVJF3aY%hSnX?ndC;0}I#^d4G_ z>4G{3ya!T_ED8=o#9`4Nu@;?x{mV60l8Wrq{XuW1?G%6~7k?PW4BLY$EkDJM2atHp zR7Sr9bXHYW6(@iHy3e)e9-4btgE!hZgPXl_B@YbJ1Gv0fM%}8mQGb7WzVCro9i_iM z+o83J3T?ob3_-|!)W=#=HAj6M*7gL^~uY%TQZ;)sx)n!$+4c4X)zvIH? zg}O($FAqdODyrpzTv|4ZhVvk|;8Wy;izHZ#flmtw$0pIO9RJ#t>t3mr1E&reXUHbT zF1Nt$1CX7CF%W$f(up8-6D1mAVJWw}GV8(+CuP_iC6w{4RV^G})5f>K>Ak-5$P~He zz@lPg;soXHvtww&V{xbgh0~L55{PeL6yWze--hR%^&rwFSie!!jD2}~RY~e!Q}TEG zpApq3=$AMbpSS-$c$e9^Kl1&4hM}@Quw=`kr!@2I^vXT=?cA)Yn+>3^VTgi+ z-E1r^ZwwYr^CZz}K4!`A>y9s}TkZRGHO=Q3sFofC36rE5Xkf6NoFs|hW5;Uf8Ul<= z@o$eFan7ZngT->@Bd4>ihM=%x`R7OgfdF}Yb@iv!DN~FCq{nFjD0v9VG^!;M&t(x{ zDiG;eImi&7bkDY%M3JW1Czkl=pcRJ^$GG+H!*fcH2x9E&Z;4_ZZJWusM}&Zs_4q=1 z@S2>Sw+3fQ3E%zq*W|$L_{-%-Wt5LT)8<2R@N1*ht#&y&e1?i`!ol%K4>=(+T z1_C$wu#>%ImxJ0C017^BW$c06 zq(|;VPn3gEb50mg>}SX3R+S&8GZ9&%hsR-SaX46r>g_(_2e-|!C-Ky5&{9r~wx$@3`m4ml9g4uDFg*CIpD{Rn z_(pA!@QF9T;a?{;-EV@(3{0_=d|*uih{5@MsIKfsGP!UVxw(hepV8W}(>Dv{zo^62 zUH>YvOKPT8aqjOBj_Dg3%NUKpD421pA^uVp2__%I=-c|wBp%?^87o9F9700T>zCjl zL9y)0kR9eJ+Xx&8byTV8_Mf=_k>v_jxfA75saW1QYmlU6%9^*#+pXlO0QPgj%j!?anlPP0na+$=t=rlh6&Q$Xo zW!CJ#Uf&D3=81krt`R8&2hJ_B6_y5fU$UY7DHx$sHuRSXQOzWot}A{<~e&D~pdz`N=gz080TOC}7}BhabFzOU?8Pg&&6;p|HVLwqY3 zHD`e=GY_yq7d-SPnGAZbZo?_UWa&s5(J!Ee#>56=O;fS*$}KDS>eTB0W2L_H&;7T*nEUt{wu7&p6Gqo3Dw zBh@h))N8$YvI2%d;s5<7@o5`rH@ z*wKrqj;k1|>k@(KV1T(2WU%W+xmG^E)e#X`iTUpD2Ur9kzY^W?z)5-_bGvpXJdmB( zRBh11N|UjqKUYra#x1lCrD9Bw6y@GFHcy~FR6OWMkyT$S$dK1k?H91GyA&BV?}p9P zvE&7T27hQpq`B|1reSrXxz4%wXf{oi#`vl}Ueq^vKf?!lF|uzxl&qk-75%(U)#h*Q zhE$R5;0N)b`ydec#s7i5zXX|rKj8n$T%!1ea{d1A>3)(F2nq0Cx&NoPb*SDdt6PSD^43BNU~Ix*8e#R4b4}Es{pWGvpWgS*UdsZ8B{5i^z(X(HE~~Rr zz402pt6JYdz`V)hOb117=IN%#CgjD+yc_`$JM0hp^x-o?o-8LyJ$AY&&+Wz;+R5q_ zxvFKLd%%0<*6O`xTj4aVIL8x8P1AinYX`M;dlS=Z%aJC8YpzvT2PFi^1|0+_Y{oBk zti*f(Nxc!0;LD`@)XHklm0&hA$unAT^8lfa?-H|2)iGs;_mftOHfhU(Y+e?k!%=)* zdN?5vN+9;s#|yH5bvS5;Q!X!f85=hFI&AoXe3pG&l`tpAKl~z@d1Z;DO-ujO_Z<0Z z4buA#n(jm9SdVOQ}hz(RJ++aunh>HsR*TIWZm;a6ENr@Rae5>hocbJ>Vd z(L5|%u!vJYh62|!``|RK1&?SoLoQ)0g04~ZR9bO z!HNH-k)FV;_>!1kNqS*EC@Kpvim)acW#$ww7}ajRWcr&QH6rsdD7e-(oebKDpIs7S z08Hvl-*|!SS?C;qfi+qPwfbS0e(0GL#aa+W;P0qAkNYNXoCN%a428CA!f+Z-77G%B`!lmvx%fVZ!EB99Qe&rL#`gBCY1Je4#>2 zS1&NnzM2mQkEL>A^kD?MaO^GM zJWu_IKv$pmkA99L;or)N`IfSe4)@04?iu<;DqJQ37DPLjp8i0 z6hOJr$IC4!{hF99cRTDQ%c~n1uR@Xz?79#rd$>sWVo5JBFHxWyWo&ZHLpTJOYV*z1 z_##~&1|sl1~I~X}v8bB;hf#%)1j-TDk)d)+> zSNafd>JI|Z-<%)R_dt*vO27SK{qf(jgOC57H=Xzlhf)98dF3+v@4U(J|8$euNe7*X zzVfO+z+oyt*8OWECyh;4lwer_RJH@TlYBW3{@~fHXbsaka#X*(wBxNN49xX|B%k&# zhHdp)+cj!c^m)ztzCL=Bk5hghOie-a zPu-)CR&+4V^b^qQzf>^nc~#2>y^X7C1l2hmaX?Q9P)^OYg<;@-Y}N5`4fIk(C2MPG zN2%vJ^1gf4&5gbKUjg5(s9`{k=!wy=GVN9g(9*-7EA6_}ATGeD{~#mS<^{#Dwrz&Q z*nzkHnxpsnGODeNz!myA0sr$Xhvs}V1y)h#+pvY|C^%dfdpr1>rbW2wq?7i|bbT8C z-T9L3XJa#UxGSZ8!S>XtqWIjYqsZmiv!LWsQ%K<=tD@SllJw8r?_PJ)#p%%1@WS0K zWJ|V-0LoFig4$8pM&ih>LqRsdCu&TqkthNYw)AM+Hb=C*&^ctwrHcXUzQ|`&vsQ>n z2U)8nl!7f2Ry27j6ez1*{NFfWsV-0MWec>(1i7Mu$4PP~6(wYiC-O?>*xnyyTGukS z?h-5UOsC76jv2B>czPJW*sB4*-1UIRE?%K17&()^GP-uab#@{tsTu*Nz!<9@m}~J( z(!rEWiiy43`lG7L85!yYcv3wQ*F@g!O-_uyghtL&n*$$75ct=`G`J zQM!i+CNdlXFTpif&*A;&@^pLJl%G}@dJ|1r?C;pprT{QNMcI*oktYVY2q)JY<{ z3>0nq+goN7Eka%22?lqXNKBKM?B9ce+T$6!&(9YU8XNl#4>0*}#J?G?x30QtLbilS z6?|;Z+>fpdbfqZ8$lf3U)SAy%o?G+H<8Q6ZT}+Yhwsqu`?~c7(`t%zJl%kY z!s1@&d4m3fMZNYyiw|jRF&sp<=sug<>E?%15smC}cbm?xud?ZND}7(0HW7|Nu7g<)>##9APnEVtN64)&5^cn=rVd zl7y9%3c8XZyBCbthb?O=a(jI33TEVE>g=+UbCEC$9)|f{kxJSWr*fU`RJv~um|zPg z8d2n7V2y$u#Dzr)^tMeWQ!`IZavhq(f?xdQ0}m@GV$}@|dJ|iV{wR5*gqAz1EDJ{q zhyk!>sgVX2XVyfKX5@=)$qg$0aQgj`3KBc8b-$lDV0di@y!e>2{(d}SB(x+&sg9ZD zsVE0#lyLgqVvdg_c(b{AQ5hZ|lRY#Pym>(8HZy3vod&N4;4QjANS*}9Y=S>;5#xLO zx_OsT?jB1au5}EIWz1=GTIK%T@N(Ku8#)dUWv}d*c16p?ab!ERT*Bq0OpwI_7 zX=>t6?;<5`PYAP9dd*jt3ke{~THc;K0;a7szw?ROi{AXq!L}l9)cl{5{7p_#&O0&% zxFDtsFlo2ejDc#bW%UZ7s;)+FE4XQ7`{Cu4JvcW78lWa2**#sz>_|`Knk{qQVaT;X zE&og-S_SIESfnll$l_}tn0=wt?}K_YnqWiP1Jhe2G;JwS3zw3}g)_<&v%3F}uXl`* zHE7he$F^;I*4S%o+qP}nwr$(CZO^Q+?Qiyeb58Q*?Cd|?o%E0HRJ!Y_s{6U_t477B zc=2jnqMEc^fptZIn@?g=T-B_~iVH;vD_mh7hEp&^BMJG{aI_Zp2IBFM-8>vZ#zsQ} zY+(?Ht*Nb+jtnmfjWqcXUa2Mt%^AkZspC>iiHLKBlomT;?gt($IIy-vH@?krD@R#_ zNQ&aHcfpdKJmiV+*UMvrC}H@Xf-cBsB}*#*X-O4whzXLzk??)lw_w>eF+MlG=>|G! zSJZk`*9IaXu@1i2IFdvB!*ELHFI@1CEkNoUK%8PKXT&aspLqr>Aq)Q}4=|Y2c*Sg( zRGjnku)&m#)NWW>Gbjf6?>;e=HC-{29T<2Z^{xiHpBf$o_JKaJ5Vix|P8c>!Tt8^o z5GujVAxUnu;+0iaQlq`X>`1u}t4wsGy+uy6;>C8>pV@x!<(bPo?R7Kj&~HioS{i&} zF9`8d*=?yP#2KL@Pbf!R;w?&3pO>mvF#paG3}U3eEV~+i2;2A=_G1uvHpmJ%@5ott zS_YGq0{l?L@kM~tv_wUJN^X1)CV6ZewERl}HRS;48CCrF_DtpW_E;!uq7NdHDTlUz zUTAcCr`UdNno;RK6$v4sy)HGmg{TTx60OTg^wiNZ-J0@CDH>;E)i%Oal?5e~6UAkd zDJ4lYG5KAkncB!G^}#oe83`3( zu=3MZ3ogdv;Q$qj*cFuO74^%Pr!INp0i?iDot5EAJ6{UaR}?SiqcOQf3%lb*Ifvl- zMz&v=xA~y(LkeG~{m207HIgjlnVjs z1(YxB2!0GwXh8UNv-KbD2MYdDW)w&nT$d#I4X7R;hfy*4&)f z^45iXW^;jRB(8+%`Om*z3;)1a^WYd4XbpMHII%#n!3>phG~5kIapC_pfc(z9zy5g| z3|^J;e|Q8`^qQLl4Fe+oOOf;r4yS};b%1N2&L`mU8U{9G3)W0QReFw@=8AwAfx70c zo;2m|EFUg{x%j5mk1C82sE0{ZOpeF-sBtvN=?qbQgetK`G|UCa7>7HoUEt;ZlM>1@ z`Krf+a~SK2c*%pR)sHWNI5U4Q3mB&RzG4D8ErEu#EBbwd+a^$2n`M-6PQbr!E->4i z(q<-!ghm*|i?73;T<|B@i%8!AzS~`}oL}aj zGgym!5pm~vjAX?y65rcaZ(5I#e%MNQ4f_WP>tW^Y=bR73!&Huy6x|7B;?%qb2j5^V z&fYvKx9h}BQfh5=q3jE*eIPEYDcq>Yt7tC>D|0AtP(@#3}P)@u=MlB_JFKf6_ zqGUdew6H%9pfn%Ta7xP1S4EO6QmuKMJcB6}0)FHg)vJyBpT$S@dXSUAH3~yUKA7My z9E3iZ%Gf}+Ba_jfH#f-pidxO2IDEQnOMWYMf4+ltq``~B^Fg_d(g z;N(@Ss9;>IRL3)^%ekXi+7yjPzfnAJbMyV(+G@xYb(weK8Gj|L57^1&S?ZT_YD3M&MZP@)lQtp?M-V}D{jNNvM*f28SB#vFMqAD4Tx*wq%D+R-rdZg-$= zmy$&S^l%eQV;kh#MUg5BXI4jOSL?Mzg-VO7Mv@Qnd6nYp@!ERUWqY7VF@s}AyWB4F zQpu9-+=A)NJh?)v@}n)efJ&;8Z1cRZF-pb0QAN_4)gs!Xx}+(|rBdm2jxu`G>canu zyP9aLi*z+3qhv`_qq|0H17uJ~!nS5fV zwX$-Pkz5U%yk)EBs;7f>Rpo$0w@WrG%jmZPf%%Ll01ZSNt(3`~ zzSi{xllOwI6uEJ;f_~^I9;0Kw{OttNqT}K<*Zv4gZ}g?<=5Q>%MT7!*fi17o$=9E^ z0OU%(K*MMJt(*(B+(U6OwfHrcQ(f(d=^df`sAo#c7@ z4b=Zwy{N==!dU&mwNYCW$vyEg5;5BAHf2P(vWV<>l3IOdU(1C=GY?GF07`fCy&&8} z!&cc}(bwz59^7R|v+=qJ$~AfTwZ&>w?|v?|DV?*dVDR`p{Mw=^zLY35lYs`F2H5+* z1ZLq`ef;j!=}&9{-yGnaleng z@YT=BJ7r*s`$8{5hw{cJs1s1j{?MH(b$*T7ZvyZ6Is%8Y;>7x~kzTWLOBzNuy~@b( zUVb6xU0aW|`viZ3Hk_fmrMX?jZWx_gRmKRxriN-!X@VUiLyJb41jCJgCYQJ=?H;XL zB@{%A&Kuh>{>6~=9GS5ypcvy}3kNN|d2?qbyU&JB;mF^;80VY~YjjgZ2ZQ5%%ET&y z=6?GbT7q#=sBPoSk2m)XVD|X3Xo^y{Fk;)MJ*K#wfJiJN$n7Hf%ZI}G{IYKjw&|Oo zSqDGPoE{b498dQ9(Xq4FUXKUFi_ptq69|o9$wl1R&n{ezyCCAq2M+dR6y&=VYR^?S zzHf-|JsQx!L7ubyE{XOF>=+3p7y4Kco9J=ILWLd()uFOy4U9MAw}@s?U31>@m6U2M zRH39Kxb(5iIu&F4p3nHJ!hI(vj*@_Pfa09cjG|gRKiOZX z=dRp4LU1gb*FoKj|6}@tPmn%yyXLmTEme}no{Oa|i0{06K|Vw0;+`}FDcT!So*$Mb z1Cxsnww)f6@a)jw9hDp{bR=S!<{CgSpl`V5eq|uo^mh>Bh1i>%X;#C@XE0xBK?kZC zBsJ)m7tuxZADb^WEKk=w+GZ$1ZeQ%Us0n%akF#sjoFg{Ytg%%szF$Y02e4^-6VDvL zZ;h~hHW^XrXuQ#6;g|1&{u&=_F_H-{WS*6LKTBXun9py9_8V_L;H*lqtn1A`0cMQ`rj<@HX>0GFZ#y;vOaYG4fJq5 z_|Mes`lAq;OHZuI(Lu5k^2+i9HzXm1$9v7bzR|%uld*;~`Xi?43{Vit4qC+f1;@g4 zO1A@q;2C6GDl5q4$&675@YRvi2E`}!`*|6|^5dFA32i{U&Mn0xPteQGCzr4k4d-pH zo&ie?5O~*DMJ<p@h8w&r&ik+GEigC*CeC$kv{u%o*7Lv9kd3ReJsljJ$Nnlp8Kr zuQR-EqL%m=hiFE!wa*(_KhB$0Fic?KT7uETqmbzc2tEE9F0Z(m6(EEr7ZV0%oPPK5 z7~*69_f%4N#=VZdE#CWS&V7}r=$lX{DcvQABK|oRU2{H_L(#)lJu7RMd_;S9fJ&`^ z1dG|t&W=Ea%~%-nu2-g~5P@$7k*d}4MKwx7f`iC1_s({IWjPcjM>EfAL5(Klf7izv2ujav>-pwwB(O!aYs%n_G@M#J-_3 zSV00QucWH59#JR%ThH08u%1zOX+DPWVn|=~=AZqtdRB2~;KgYn*s_iek94>yQae1c0Vgk4VLZCFOX}m61b+#~~o4HQOsJ z;qEyke0M5w*j&Ldls5}NSm7%mESXr6+c-;7k}SlAc=FYB3egswh1e^Yfr9^ZB$ER_ zP$|8jKrzdPLUh(PH3(<e(3mK-#+0h1*s{sROPhcEU&QKJ7qMor1qF)NB6>tX$Wbhi&h|6&{H7}ehK6?;7 zx}T!m^*-L-;O@+As~l!zeCrekGDph0HG~m0VYK3G3PhpJ8g7%y(*zeZVk=J^D*I>y z-{u;S57D|`9sk-Ze~?bbEIC4GZV%{ZzaOPOe9EBF#t+YUxe}K~;v=7OEJXm9 z`XTAXkx63!MR z%p#~Tet-7 z7%jrXTBh{GKXwjNdEjbxb48ySZ?H)=t{x^h+H4$V!*IhcD|Po`;8pqRcBZX_7WDtH z9AN9?q;q1WQv7{zycTgNI-bbW9iiQl&4`Um^9E4@d%>hl9&qR_~AFX_i_MBt0GooTL^JU06{h zVNWDGgsfGN;a+&)5ss?80ZPSJI)zXm9r8{rsJ4FTW!s1b%?~LSGHbeTYhemrr=|6Q=JNS(xi^5@MYjYVS*zR;eV0nl59p82XXnqa8{q%#h0?tbYA629DVlzB z3atNY5BNVCm;XF7GW?&QVuKUuzo{df78lHhk>8{uAt=kB4zui_WzBJXkrJGH581H3 zC4F*z?w)uwb^r+GVbH}KRX8(t*@9h{!;%YS^_u%HVMpu` zhlc9>S2vey1w0)@EmcC5(rU{^>OnnzVE4S);VZ!l9_TN0DvUydjL7+$ROF!ri9K2D zxw(M+3!!#JMl#1AjOKCOD3;y}lDXYW>hWS#eI9{K^o0iQ)?SbRy>JKat^sP=PJ$9# zb)7n052SBa$UpJG`!eQ`Mf`HU>ipT^R2%VKs@bx}aBeTabS^G%iSj5e#`F+YzUU}2 zSEwdm{eeF3j5$Faxms9yx{(OZbP-2NPSH?JE?J2i|8;9mIDcd~SyEQekb2w6wro~+ zhN%?kN?R>sB`le-Dv7UPQB^HCcQRwucYnlmY6;h`W&I<}FkKlpsN4a$V9GcD5vbKB z(%=&-7=elvB^PPzHmvU4=K5Z6L{dz1f8lmbd<{j7;VUgj-(ezm`mqx+izg1RdRYo& zAh-}w8qXT;z_#K3a6jc{$1|^4A`$m@3#HW2b~Sky{>-1!2nak%xhtd?ZzlU%Br2@# zVf1Rc5F)OK(pe6$R#J|@srYcQt)=GfH*4SBT%jUh zLblm>bMC*Ki@omJ@F{WWV@4KHw1+btxKZ@%z=wc<0Yk~t9xgoBr|(8yo9Wu==)=EB zMWt}QT6QhE^;lAZ))qSBhDUs5ulNkX(Ah846K@PINj}XGrTHGyT0b|EjcjPUb=!J& zd0ok_TkZRnxsvc&v?82&LNKYcUOO<{rPUWIqMw9E6-)=kNA-4AlJ%X784I1dW^VT=S<86wN;w>WbhDkX1cY@^+3U0 zVhK;fU9uaE+ctDHBtQo~mE{S))bDO3;6QeT)Cr}E=V7FA%gEAf2hM~SEIV$=-n>7T zl`r~fiOZWNUzaVqNgyNknAKHJD2|AW5eAUvXRlUZKCPC$2vE&tuXQl1j4&lWx|Rp+ z=^YM)uArauc{Tx;Ez5~TtR_%!E{bxNo<6d;wMmyXOl_eLFPKEG_z~m_yUp-!#M@yx z`ehMq^~^`qDyy_)X_(CGEtEtwD8homyVhXbVbARzZ^k9IQNZQHLmTgy8_5CETzc45g# zt?{;vIHh)(ALcNy?E{r#(FEk>-x?#$v!qy^fpnHBw(n`y1Q4hL&@(eb0;Vl?{-c?z z&*li~NH$aFs*yDHK${bkLxPMxcBzRY&Zt@|$3%6OepQ^g^P0=P9Kkyf1;!1mt$nBn z6FsOUA=V^KZM1Y@X(Y%jM>X0+$XR;p%21>6v2<9V4a&s(CFA84?S7lY<6qrHG_qgg@_w zn^qN?Gj z@o%>gnV+_iRyC1e5bX87Qf4PEc%WAR-}D4E)dOwHrW14wHIRH|PZqd==v&5I`nu#m zu2O}Xr6i^&t)4-9?8Pf2=KuR+68->xEqmcr^3loYjwzY2GyoZRPi~6Tc+Fkx2-9!R zI&<{tB$g$2iYa|jgOY!XiCLJpO*5^R-2wtiVK*cSK>9DJR0~!HOx3^J&cBb_k7&V! zNdUWnUS}LTDYxHE$mDixlZ4VVQHd*)Q-E+#}VZcV}F3-rPJ2-z9$i+0{l6Fs{c3ulpURi z9WBDzz}3Pda#T-WOaO!^0VHMM^ho!clLB0<6afyHJp;cFfO~iez$GxZ-w%D?7D}(2 zpP9xE^5OWwE3pcNmE&89^a_GFggbWq?GQ2WEORC&t}48Ai`cl?Ig-SLc{>~D zl|DM|_u-hr1H9YWkSwKVHAr1j=3XLq;c7+5O7`IgueayuN*f=%P2YSUV~_9h`OHuE zlL&L(&W}O*&649sp-~4aVWr!3ER>ga;UH~OK`Wh*A%_WU%XYq{y|>xl!`ODNiSEa2 zp9Z<2C}aYV+FLje?$yqASNgob=)Ym-R^(B(Gx3?jw4X7DdGUc^xC$-No@L`8Ux|}O z=Y#&&6W0OidRP=%^Bot$prfNo)_nd&VLBCgsv(bAEIf5l1$n3U{o^eX!HUom75|}D zJzR)lpve2>+c%_zcZTsSXOi6ZztC?&3rBe2vefWiF68gA-ZAld8H)7H6iY@C%o27>Yl+-1DZn2T_3=A?r2v-o zL-hPFHSaV%y@IDEV&T1rXJanwA3bW`3n=w-6O=fh@Wi zB78a<+?h-@p#aq9Y;BbQhl(rN%=bxN4WI2yL;n5Q>GYVXJ^bQMiIoLflEUv%Nxqp8 zJhYM@2v2LuGyJpdU?D~8u*7C&+DnN8DOS2%q`hHde>3{x5o*#Zl z7JT{l9Zb0YWoiB2j;bkD&Dcdw1Rqh(pZ>l;bdYez3*%<>CW}sV^UXy;L05MH!V6Y3 zaU4=}{kJEnn&wOtet3b+RH_-bk%zZTTQbv|o?yU@1rlTKhWZkOD6^wGaI|Xcu5lGx zaExxQpOf|VC26LND7U0}3Q0t(NCoTTCq+^todLz336$7SD&6Wd(PdiT zlL{GTZCGuzeTN(Kg^g)LEdO0pSphal)SN16bi-?F0SIUrFGLP<1`PV!Kn$&Av6 zMnqU3=El1$;TH^A$E=WUf8nTnV>qd8hQ1eYY}rz^PD#g{2ommlr0XZ?rENiO9u#r

Q0nDPJsjPG{ zcNr8ZX=3f&Q=mGq;l}2jdCpy4p6)eYU+D9kdBghtS^$e~4x}BuFlt%_63?lXX$SY7 z84Wj16ZjUfKnIDmc1E8Z7@$)h3gAJ8UtGCUS>xS~u1s9+q@0~4{Fk?B?!k8n#kpv? z+QScnC-l;lwr2%bVWY9)z@865GF_?^C#)GOTcX4{L)?_?6$DOzTa>INWd$dJM4WV| zZDGJF+GVeFC#T!j4dDV|^)J)}*<5{k*j%fLXGZS!p7#c=cf;5Pr~kI%k$!A}Ay7eP zB@4!S#9a8hyBZv71dBzFk(Cxbzfn0BusYgX8U0nAt(8>p3WuUjaGNZ>_aADrFXFLPFG4-XqtLTmv)V2KTa;_F z?`fUg+z(vvDak1@*j>E^3)Hj|z%weAn(w79#H;~%8w|s#lwyP1|Kdb#sWcoCS@u?Z zfXgoAV;Zo6vpoHXwggdCddAxYvP82S>nOy;Ozx!n&`7;R3euLyw8b3Y_arL4eO!P3 zN(=sneRdy&Wk$vCWZ(+wMtZx7dsAs6#J`i zX}xEDoCci}&0{#t&iuG~RXs*mhazDj$vg3Ij+Wz+WyGMkh5CnO-sySRo+Y72f z+?hqzP{#>T-<6qcXmim{DT*7N8kZM?@ef48Tp+B2dBVl>_DI9%i-Ah4*|S{gB=Zp-vD zsFYhk-Nh?%ubI1Rdk2PKvdVqe=dYW&=5CNxme*42&G#Mppvncb9sT4Bpbq_}O|5k& zn9-Bsn8aU|(m2s3I>q4mEQovP<1sr1uqJ6qbDw#ZE7b`f<26}v2eCn>g*czxyGEvG zs!(X;JS{c^-W?MH;DZEwq5F`Yey$0*{Gp|4k#A-gESFzJ=oMPu9DG{5^OXj5KtgrG zz?=FF6COMxCHk4(sX8)`8@fw?Q)F6=@YCn5S3Gf8zZcj-&N7fI2RQ>&$x<>H4z# zfenJ`_+i^Rz2R6y7z#za>Mp_u8%!3-^zP^IOVY_1GqA$t+kH9N!>x(^U=2%UYY7WM z6M>Qq+h~JVj|oYb`m%v07w1{lE9S-l{ZTXKrd_Nkz(py$yG_fr_A`*d8;k9A?l+e& zBe`M|@C?=i?rdw#nuiCCLu80>$iyF?ihtV-X(OI_Ok=rz5=}|uK|(P8Y9-4~x69oG zyTksxq@PVX9TrHi_VB=k%TD8h)t_O-jxNuSud2=7%`CG(S;=tf4%lY`v_Ioz&b?;A z^F)uP{M0xf@cwC$9z+}2t+t!lj2f%Jt9McBYRgt47J(R%2Dqq|#%_5e^h-Iqf8}YB zCC$hLoZ3S^ZAs~Wpzxe~fD;78Us_nyBWm~L*EtY%dCn%vvM6|Xkq5GZDBpHqqH_=1 z*j>M$KuKo>v20^g3)bX!bK-G@goc-v4c^V0izTS0BLqOy6(4;<161D$mN8>`v2jAE zJQlMjt|rV8K+AlB+y2#-ZvM}H=}xRYC8J-7F9FQ|@(=&-iuEiN?f*h0y#EW8NcCoL z0yIPx%Ya-b)PrE!D;`0(U%JBr%veiNMYCFv`SGkJ3AAQ7V>S{aIy=w$@OFNAV!2J` zDPSbi7hS%9Rp!qZRbk(9TEf*_wNX00_^R#QlHN)q+#n_tB#cCDLmR<$o>X(HG_P+J z_QzfYzLZBwWU2HdnH%$n<|(>j^s^KZWNYdh{%vL7S!moO5g^>FfrQmUTZP;vLGV7f z0?Gr86VWUjbHyyfwzWrrZrWu3ZB>^ijxZ2<5$o>FwBCjo9@jNqx+$Yow%;L&LlUj{ zGeVeA&xY@^QSqm*C$|RYabjVadx*~^%bH1*-g2p9n!nLyIphZ~%3pmETQ$VK z@S?-M`Um)t=jbD1S-zBAt74*{5bMcOPH#tE6fMqqX}`e3`!0+VLHT ztUNW(0_~k3k~Y|}4KAF-!seww-|)cLD~E~B-esj+LE~MycK1lo>DKI7fNkql3}bQ3 z2Hp#2oL%q>r*Lx&G;UI?WbzMd#GS60fC?xMw{9t2AW;xfKvV+7Od$s&`nG+05(mBV z!;+xyZ<|-7CmY8w;T3pLfpi;EXzqN*4+Jqt)70X89(P8#bFWtUj8y5x+P^+6nJb5i zzQBuaD~k2tO1wKrT4foMt9`+Bvu_>g9@N*Yiu+p!B_mqiAJO>uG#}RnQ=S`>e_BeNz;@C z(rJ@M71`Q{-pgA0WKrB)&GJWr_}&-d-u)`7CQo6#m*sS7+v z-SRt8LAUnBN!<|xp>cmWU?_`4&4kJ@$ee}G1SZkY4N239D?L`)c&B);6L!!(s3VL} z|3&RVG8SD!3Eoc6eO(i)E0-2)5U5tbbsVOr&xSu?xm%5xQey>EKT^?5u;lQ$#OWF+ zqzka;v_%~R?wM0(0OG*N@TUD^pwO3DKO)yHR!>gXhDP5A$+I%qW+)~JD{>Te))4ZC zL%Q@zl{EKr3kn|ZjgyTOQ0so;-y+zFB-{xU+C+$u*9zMuN7MftCep!W@HCC_4!2m<2XO+=-2RHr2duwBvJX9L{yU*yqtlM7-m5 zW`hKawbw}*T{mb_Sz*G$KT&$eK>XTo)iS^(SEeG>3`UV0gpt}>08X?YS(@PW zBvGs}fmTZ7-dK zsC2d8UUi)N_VV*83=;yogV4(~t+M|M`)1hxjy zLav3Kvbi@~AECWoZ9Q0BlPVW)4))wlBLD2-XlGJR9)^eSZe1QmAVLFsKVPrd_`1@x zjm917$I&vM0HV<@4)>2u93M^!3!m1FU3fHOYfelnk3HT@m^d|c@M(9u(sPYPZ~mRW zkFnBzoO-b8!nkkp)h4Y>W?K3C?9!;NHcopqX=$TqOQo2$u415Nr+v>;b!J%K)?ZA% zO>^E{&0b`#tGjS_sT*Hd{bGUNuY=>qmt>mgKX}~fIPlqe>~6fBGd{d{JySERjl;vr zBe@XpGJHFErCo1!w^l7XMz3;{xNtr?Q$HW9hCE-f__7P)Mj<0cIPW2uWG#pxcS8_~ zYvBbX5>rFpx?VyF+S;%=`0%<@=(Qn%y0@^zqme+HPFDyQ`)@n9Vs01C-zj_!JX{Sp z^pAqOP&DrgwGR)|#@{Jy4Ya$C+_O^LpVc@$Kle+#`BBI zc-xQ`d=R{mm%uS>xRTL9@<_>Mz#lnBqs}7J#UUh1jVbi?+DA{RNhjLg??9EcB=l10 zXx%B+HBbaNmARtbgqwx&Wbt)6DNr6ktA`>6sIPY$wK#D=IG4kJrts&&#Xv ze}~E=Id^R7N zXPIp5fEQQv4YrB9ocxMeydtZ6V9?)e2p_+n)?a>Wb$f)mS(h>JMrcObNbIz@mRkH+ zj0Xe@=t8)%VsCA{wZf4!+JmFY;d_LwIqE#tW^8IrUenB!>a?Oyd?ap#|5BC*!?2wp zgl9T=Vb%D~UKN&~hb%H%D*3IZy}RO#S|a|De=%ITwh_xcJo6zmS~RI~J^vhY-po18 zYvtn=;Mj!4txad!^YyQv#FI^oZ_9}Kwz~0>Fz$uI_bjw9YZo#Nts;%pAtDrH<=#{~ zZaQMh>;No&e<;ShO&kk5|4=rn{;JpnMa>`mL&FGKw2yfg3Q)BL*WDG6)zyeY<}|qE ziYwn3xD`_D8q%udM%2Yf1CA`Lc8N$jCg>QgCEin3;@7dBieS;~zn6XZ9=$1Za}{t~*i^uwzZ{BFTmfr*2n<`Y!I8#*6YA zHPj!c?pGAek9E)7(=;dbNuG_K{U?p%w5LT{k{AAPD|%b>%r|G&*VXl>>0WrB_h?GX z#@&Ca57VP6X^#0ms1=)$%l9;zwW7zwr?5xiXa{p?r#3C=WafJsBP8}#gVA{{(T0<3 zWc$q7GTP)%UaFCHD`Sm7ehJ5$H8&k4cn6p9$G>UUp7Ji>0fg=HL?%l>3hLkX2jJDF zfild)&$$?ZSvUZlnzPq{E!ku?N0bnLj12u#kP^)H{M}{ z>!WnQp5cj2`z&Rw8D5i0)Ac7?Ow|Ms>&qECHB0{8%$^g_HFo7pqx03*tKoo3huzMT zOZ@Ff&7?sMQ-fV?_-z=|+@PHkMg6iIU`^D5GM27iE*X|y!Xx)~d~4ri{NRI6DPL-u|JZ1HDgt70E0}xop+oy@pAg|*~0P9 z4;Pbqvew6VfclhPHANKa(WIr{UvT27Cw$zD|69L*3Y4|Byj(cW7s_PX7wpOvTkbC*et6g9@CChdKkKU+L!k{G8?M2 zZZ%YVr_q0$S+vYa@#xH&Mrym+V#6 zmrfopZ@!t_Ua_5lySb+cn%ObC**kkO>VEF>{km*=)ND;|&8l_OO56;NeA}Ha*E#&f|89feM)is z21P0;8Y-utGAIzX&Eh0M`mK?KN5gOpqV(NtZ7vq$t*JZEt4yW$i}8QV~u{1nSJiDTago z(wI(^S)Rn`cXB<)nL0eNuPg#=ABma8!|7492El zQv!+7@|N$_LcL^7GS_YGcB8fq*6-xpicoF!5=r8L-MEgN}6Tzar^tzv6#z;Bl}{ZZdyu-n}`J!5goaQt@}33I{sykEkG3wqg6y#fo& zZ#@SzhUqrCQ>GQ(}l%W}$lh~-NB~EQJ z;Q)UtNTB#Vb6%na%JxI5(CS|=j}UA&qf{1Ss6;7ylxWRUmF7fjs)1eWC?SDcT}mEO zLgKGD3FVL|wWOR^vjQEEJZtfu(ysSqc6n=3#1PsTfJ1G=GWK`%i57rAM_v%oG58S> z1UFv|EVyh@;Bskqm3Pc>mz$zh|HM1=b1A~IRFtDMCO$Pjq}`og#^}7V!vionc>-jP zA5j(y``4T~DQ_%1sfJQR;l^{gk;qg+TVFP)9(Q)iwgF!;5>XeRY?$BdjO)ox999-= zHiJ=>p$pId#S*0a_UW7en@x#)GievN1SudZncN(HJ3j>IS1=VTH?~I^E7Z0E3#-zo z+8oRooM6>7t|CfLLqtD z3@o)P?Hanmq=GePr7E?`77WKP)~e&#EY#Pc0!Yce;(SbxBoVM9w55Q+Nx-@p@-%oyyH_&Od;7;Zn;NyJfIsNz{1s?IP-$Vo`{FsT zWW_>ih5_|wg#G`0(0t_zHaB@h)#2Rh#Ut>{S41PBh_ue+JmwUDg zAyeC(z2blFnEq67_~w>VEy)t8k&{HTT6n-HXqwwKIx*bN-Hw zORhow?Hr?Xz$2aI@^&+kX*ro}t`n?-=uk3I((j}x_g|CdJ%I>9Du;hO4s};1U3Ale`|9maM#)CSb&@+HDB5gVV7fe_ZFGXWS-m;&K`zs3bco%lkWH zD$15exmFle)vSEo@xDB@(NbV=%DBYlGpOs$2Y zg3eB(le!xf`MC3)Vw-^XhEnIE1A^A>%FE%0g_SstFhsn4E}bC2+_g zGfCchNIZ2Dyf>q@Uue9cEA(j}Bq?EGZMQ}9BBs3hp*;MvcFPMHA)Qw_cqyqnYe+mz<+ZDOV?N#-&a%VKx6f-m z$BHEm``F^RqF@Y|?qHIH4f@Y!*s-`032-%C5tg|^izcaX>N{FQR};{S<216Dk~XoE zCF6o3ftph$dC9VMy=Wu`otL}}dz9!asL|;h7d^&Xn~uoVmFEiBl1#a+<2e&8W37EX zk@#|-!n;1QWxC6DBF{Uuz;=QQm&sL#Un-4bT_PII4vVS}EeaQ$7XpXjBP zcB{WTR8cfV{|d{qnkn%j<=Sz*A1>cNo3OjFY989O91;qwi1(O9O;{idXb*}@3k|Wi z^>e{(&()3Qu!$F#A#HeYc`yrzanbz<(UBeK2X%^7_GyBndu`T4yQLw>GZNA{6BRho z+7OKKz*vK`@XKQ^g-eOH#Yx9aZ|a~JJ8GF)pyD5z-mO&AKTZdvq z64U|caj580vu2to%7>v%&X5SL#gWoy=_U;d#E=x8l2rrz%64Uz`e3&o$IZV)%{oG8Xbe6Ja+6589L({ zz|fEd;L6Z3mb}Ok2d8Z&JT_>knp;>FG@KTT#ZQOZfsk6$8D}W>N%6P2toCcpS-t=K z_o4HAj&ieB&P5z_IbZR^)=CkV(-|n*x=89Z{vv|NRQ)pCHVM?|C(*d|D}kDTMRsTZ?&{q&!Yw8A7yC$LWKD$=JJ<% zDSOm4*kdATtZ&eVvjsGI_Wn+v9pPM5+{lls2WJ*qP9{t_nW;JHp*;Z*pi=#@xvVi` zqDPG_D+~a9u+?s*IZ_M~kj+Ld-V#iVd~};asDYKa^b|hy*k{s|-z`@l`$YcpqHc*HIh)kv~*l zy|XUKJRku0edf(3!YGC`Gh6v7zyMy&k0H7ZFwDN}@2F`(0-9C_Bb43CLh!HrG+)S2 z@cZa`>VPWz9iR2nh5^yU<2v3TO%(%FRuTk2b(v@?qRx~+lpz9r#BS*Hw-e-Q<90Z0 zAr1uH0N2?EbD3rOlB53eMJuwyFm9WKN-%8lzLIF)QiA*&?=z{5o0?JG!4Yju)vuM! zZN=F z4QYg*wCHMWc&SclfzTMikk645;?IgQrC z+dbhb&Le_VbuMsSeW=iOa_GWob=)=qNw&MjN%sVml--=YzA|GrUduyP@4dA#whk(^ zZcc3rp|INOtQ%~;;B)G6#gCu;5*{O-+PqbGg}ZE-tg7&FhUT;1zjx-~{&2+xA3^;G z{D-r%=k29mwzs-*xpUq2q4MpY;l(hMlshs&oVRHXl^wUyJvWBx7aOb17u!Kh&WNFJ zR6rn@fe0cYlyzSiE{U7^qF&ShtUtc&)3upz(86&3;jybEXE6VMKYOQN?Wye9!RPWI zv462j0Kd;a9dtJHOm@0Ux=dvK1j z#ogD44_&1G zuoXS>;dA*$owkC__TG3RQl0yo!-vCNZnYU>c4kM1;M%}GC-Bx4{>KQ?Q-_$7i>zoR z#{Y*HL<*_Pmisc=bf`**mL_LYh~%LH({6HCJ_@#4ap{UKWih=;qUb zMy0^JXR6$=AwzqK5A6;7Dz&=*9}KJVE!;cgQ59=J78)&!R-7w+T<(H*cR|I$@d0(Q z*8aVGwqIWkM4I9VJOA&!eFg&xIUD(mGJ;1?qR69Rh4tn|HMs*+`lHrzK{?9A8qFlB}%ZZ>9TFxwr!uXZQE6+Y}>YN+qP}nc;~*=>rT3-J&PP< z40dL$6&d;O?^_0lKV~icHB@O$V2tQDHMB78c8YDKSPRW@EjGve&u&fxlt^#c2khB~ z3rGBUQG{DUONLP}gP!vKO#sx@GA;o)FinG4eE1p?tcpbTr{br^jqKF{@dgoDZA$!2 z97XZS`7EK}qk(@}>dC}6CRArlO?D1f37s!G;Dcz@Hap0Y!~(&Pq(-A==T_&X)Wkp7 zgUP>yg`(U^S(G0(!#Vc=Ki%ZZ+NVXcCe@BD@F3x!26K2SQ4QiLF*b7j`ODhAMVlt> zd+?3kt<>CSF5;CYDr@XDB|Yyc_7;25fophJ?XEnipM;5MF$mCtDO!JVZsPR?VxPPw z68c1kU#J9teNUojmEzQDw`Lg5>1Z?MlmnuCKs2#L<4wA~IBz_GXpG=4Af)UB-j@T3 zR%(^oZuSP0fL|=JwBR7C6dRo^9WkrceuwyCqH$hNOJ7Vfu+F4EUlVp;ynTX#7N&lA zjd+V2;1L>VhQ^higq@%}OpnhUkoWh9X_Ae2^)r=A(U;fLgnxn0*Cz5#N`}|6gq>Rq z9J9Cqg_C#@&N?ljP7i|n@W#vmVitwm5EVH-r?+J&T-mjRZZYn|bJ+n#Lj(EAuIlLA z(#HXi7^0EP9(mNaJ0qvZ`{B*uqhY7y2^UAJwxn(jQY0AC**uos&RmcB@0Pv;`e1_P z6=g6in^fa8gj1MW>9Y*X$X5bX=1!qF4u75cC=cd8E?y8-eX1iJ%+->#^Zs@^AmPko0)>+Hi*pVcf z_R6FUe{@=H#<1}V{RZ7DHmHMO;uRDl=+uDnpVaHK*wf1QPV4V$_SVaF(U*V!b#RXV zxDdkTC6N(?oR=d;KJIq(-E;J*B$izm5E!FB^XYH19@OJvm#4}Iqn)F!=b**xhX_be z4DcW(92{EW_!mt+>h#w`dy5VcMma&PF{*iXkso=JhL0i7EiL69^Ki@ddJsnMwQ zFF1*iI7yKp=k~gakPydR?K7Fr4vH0C6u>i6$08-)DPCQxj4Dt`jK~BYK7Nj6*!q+4D*@x!9 z`}M9Rf_+t;w&VNw`n*lbk^Xa-$U>px4=IS$DYfV|fJF3N(^4fU2t8p+msrZ+lx2jEqKhq79JocqCK!C8i#DUbQS?f*rcFD;KznW ztne9r7?Mj8O3AY442dOt*iQ*+gc!{{!iG&l7TI76_Sq&D5eNX8Z6%QCNkY9)JRsVf zztb@(QnZAcNBx@UpW5|3Omapijs(MIr;lN%487*1+VAvsh9N~ z-|u!X3z|S=5kYGQ1Y3maW0l!0%pD-35;`l9VR4mOPLqstCxhf)N*l43lR(I}W~}^tgGa5gr|swY2egR$2{KwSZq zjp7q9$zc07HUyWi6$slu6X%qNa_<=EQM?v_^j=@v+n8B#|W3-&92#dX=Ay(gT3kNv%%eDi6Dd!;&Vjwh*Qji zrQd|K)NifvRUO!7hhaoSaW5?#9yS4K@4CtLHMVP+ zMiiU8m#JQ%R8_?dW6s3&PfED~N>I&hWY>8oW}uEp>8`HL#y$FjexdGE&@h+B`piRN zh4+YLp1xDfcvnvpcY{>OK7}|=Kw$bT<1)drOkzOYNJt>U2t0wXdIa>oeuW^0Sv1sx zi9A|9JJw9*PBbJ~9AqTe3+f|1mpBC~1~GpzW9K>^@6>Bm#3H3ZZ&-=WjOpz7P-kf5 zl4yt=M{Dt)XScI=gg&cX1wy!t4PU&zN{S(&DlU~U2QvL8F}Qe8V8jDK0+Wjw5S6fb zkkzAuJ24KVDgudkLXz+cb44fUGZBtcgCVHzpt1M^HvHz{ABi6-^dbiAESsg{0MA$g zl%QOesgMUyoQKc5oJr}(Db|We`>6$5GQbw7-{TpqmvSTk=Wolm_k9pQvf*}@jA$Ug zkuth}s?E6OT?(vB&r}8Fvx*{^@uD8V3e+S=O2;rFOu zohE5?T7Sne7pr)>J1z8cBD~CVXB_Sj6FHf;)9j%z8*G0k$q22-0}co8ZeKo(;3_f% zsJptskPpU=M6qIMzgXlxAFyz|h^iZ#33>!{1Ta68BBOzKy|zWFd;Lt;*&*n+FmLVv zkfS^(h*L|I&*}rvB~Ok=Tz71R?l2XhMIVM^A`XW)Q}hnoAcfC`C#1eTWq0C1DU60w1K`_}d#? zV}Pfr*r!qo9sTRulcZnEB52nL`6AdE&9-j&!WqkjJ!-ywImC&;LlF+Vh@ZZVVqW(I zd&Il|)B=0I_b_B3=gOG&yl?`YtD__OV(63F09xdEzPKWTOmmE|2T%$0S47M_n(DpB z+BR>B{+HRi2y6D%XM z2rFPRN-9DbvN&&>>QDlVd|T`*@Uc4axLiJtQbX9OM<^u%qE*>G497W_(AWqB2qd8* z9!p4}V*y@>nlx3SK2<~@GII4Tz!N}gB8-l?s0ui)F$9_|MAC)6S9OuE7j-Xt{v|qL z6>yP9JDQf_>za`JKkbwT9Qjr`$suPTu@u6v;4%0^S3- zNC<0bd#W?qSi}g$`Ai5qsJX3e&; zrvdu}nhK*E47idtZSa-BrWWC2@&@@$X#>&p$@4;ssbX?@j=@XSuCX_}rV5f?cl4PI zxrKX_)b!MG#CqXf9HVfmFjZqKe1JemcSo*;BMQz5ArODV92Tz)9w`zq54XMla(4=) z(+u=<_zQQ5x` zE1@k0vf!~f1keu4DfvZq7zodWvxht9vYzF&ouE_Kg?Zdr@~{aU*G@1rB%%LJ&%g|V zDHv{ms}B07l5361mf#u=Y&}b(p9-b~JNI=ucA=%OBm1+4Br-0Wyb5|sP&MZ@jb4WPB2^bh zqaiK z84R505cn@dyNS-dLK%Zw+eP%B8hRhkN7HPZ2n`2^qmP0#@a(_ORQ$paGA54e&!X?g z3!T-Kvl$PBAmyzW)py9$nf)86hmf$yOVwTJ^mK7zKIyH~{5PGqA%RcsZ{LrFX+Aih z89FWT6&2nu9p#7Be@kLayV&`jzdo@$ZUsV)#mAn4PJtLt29<@ac5Dgy^ct2;eb_o9 zY8_YZB9+hU^0C;*>y#3^Cf}?TgB9}LcMG8oC$B8E2*n2ACjPnS19n1nGjOiIk|2vV z>(NmIS-Ov6H+n=xF%6>MTdl8$l_C-59*d{S#0r39$Qq|%mdzdY_bHfj>}nL4kvRj_ zm-U}d-{0@;pO0{AC|cTzP^AmZUga08C^R7Fh4q)+vkrU}qA+^mhVKY7=$#YhB(hl~ z(zn^9o`ZiBwz09Wkd+jc01X;}psr(7%1oedv83%p&(V+O7MhEB3`rWq0SQI%0LT1y zNdZdgBYhoI)O~KC@%WLD96(fx$FH}6?nLa*u$Dq_w0@)98yIlq$TUT$05pP2;SHYu)68I~gWTxd!@wJy zQ`igv1@8?S6l;d<5s$TYP zP~t|2q1I1m!V%VimpxHzgk&H{==gJ~YEwOn8lkSB=sJlgCq?ceRwi|~!FN|Y(%l5B z=#=)bt6G%Tg;wnIEnj<;eGKJAFz!}MfR5e}#!*F-%%@Q*De#0q&P5>;q^^qAIpaI5_qLIuw&kQsmvrXAKv(M7WYMDfvU0VFjLWwnj_arbfkpkNU}Lc`AeR2XOr zQhLqv6jghP=+cS#nUVF`i>Y{El+(jh zGfTqFqDHUin0Iq9;&ThuRLrCQ!sio*c`pg61Yu|>=L;WEOHl&zQDqEWrhr~6Q%^=T^;Ozxsq9c{O<*IhKlx&c&PJM|K)k0YP;SAbX215~-g zoUys7IAhca1**bICIRw(x!#EcYmw$@b+# zRqF!=K|`?#nd&pCt^Ti)cW@>RNoQl3bR@q!4ag3_`xAotH(Q| z_}BTR&aKMioG%I13vo9IF^Z-fLbjoGo#~ONQvRRk4_P)1g%{R%$402>L*x#su2R>YL!HXC0Ta3?kWs2}Neg)y9|FgO}tXFIf zty}H^?OCr2XMifAGXUc6MWJL4+UBTl88X805TAVpa>x-~we;s16dDl&BVFDD0S#s` zJrqXRFpv0bI7_W8EnrSCh=--4R-P3F6kh+*=rYioc_SS_*L~hd?V-*JPF)uDqX0tJ zRCfxXh4-x{j!?dc#94rzMTCjf5gIoLRW+5;d)-ygJ%<@7um~h}_%kQ{~u=Jl|K2JjE4%!3|k=~#H1<&#l1ZYCx8hUx_ zVNIB%r~ohYj2IaZCITgZIIb!BiyWkS(SQ^{_?Q7i5g_c&f6V>5%4c5;E$J6r%(Kl9 zj5R3lY~_AKudG^L_XmuuMx-ho$TKsAIXgQx#rU@#o?3 zc-QvStUKGjY#$+u`v}NLtK^>d2*Rxq8E%r9PW<gUjjVkG*SQw(Q%LY)HsQ`^Gv<6j&>bWT$)bwCt zP$ibWk9FC#W?>euP_kvw&;V3;!iuXyq>xIMgO-c9RnQWBQk;pxtGzBNJBR9Y>x!$0 zm&mVI8U5M*1C=Uw#5u*nQrmrle7*c5LVv`SCE1p7tWf&aB3ZLJ<@H$L{V-4n1#wtr zFxt9?kIi^lPqK#hj^k_NKS;-tlBhMsgaz@T|7kO>aZ<%rVQl8G{WfO8q(Te20lh+x z>H$WLbG4X?gr@h7y2Qp{wPc$BOp%{{287!tu5Hwx{4JwHQ1q$h2Wrcf0xn9wq2sVJbeGxuiAo>|`m zX!>~8WFeY=YIN#lA?^K>JUqR_WLQ+axLB&RM9(FuM{2cD1xh&dJ;E^PWfvf{rz$%_ zPeOG>ysYUjEN5kJ8*^eVlC{mMEMChS4n8I`P3+{TJ>8nzxKtH+xVNzjp#<3~OWp}w z3-sEG=dzb_jf>sNrNS>pkFCJGwiYiU6&XV4B0XUy5uNu46^Sp09`{!7$j%Zvgv4@P z^l56XYKME!)7@uZ*pkZQ=W&x1o0w3T{B^x_3QwLRnB?#8Ve4Q$jku}v2KVITKm0lm z=%6zCj0}?n{^Qra-eV{JQYaOnKdfcrR*ALY+ZsA^MEJ}uFTP{Upo@XuYK>N+JkeG39 zYb9T{c{NT0Y8H?Mix=U<&3GkZNVg&g@7));rANn``J64;EtK>k8!km(?v`Q~uGWHW ztmn;iy2R0LnmqCR*7-b+83c3!__XUn#IJ~Q@}DdaN@r{B_U3R~?Gnn@*@;FA*OQ`L zT=u8_0$gqz^MzQnvAO^n_i8B*bwBt2@9rnnuI)xnKmdRtuzwv;|K$kT8`GR!3S>YD-Ua`JPk1F2)7p%-)EmjGE&K%3=(0?cBpILhxOFqY zoyzyH{F@-cb9&@+)VAq+&d2HhIFXTHpOF9()zr*nBe@ep53iN*Mj%7$X_ZG}c4w@& zy?d>jj}FNHkd5UXKsAzD1a07MI5i2^bhiYjey9_~p4+0jpkbFOP`rFqbO%5&u~<(rhyjRW39%|f0a=_%>+9hd0STGO zHfv{{Gd#u4&UUKcmNUKfN1$1@kc3C;y}UmNvPH%sh_3{o8HxWB`=ch56J&CD%#iWT zaB;65W7v>U6~&nuJeZz6hKMcZd5WV(6>i>jPdz1Z+xeTiA!c5k7L7Qz-#shbffnQ4 zPNQMIxAfy*VgK*Oj-|aWho)~#fUyn^0JQ&>-NwY&!hqhw=3m8G=uB)(ue{CFmDV?2 z4+FoXo;(?K9G{sCM?!?@DGKqwck;6w!lTh8Jb?%n+TWBs!YP6C^t1;|C_Pjvj& z`MW3I!s+upDXSTO^Q#-GUq@oqx9+wp^Gok7U1q}nmQzg*tz6oVPNDc6$D%&5 zYKQOym4rRr!KCNc^lOrsBW>nUt*pc*52nJ(4welos91}Lpa^({grI~Y|VFkx$1c&{l*V%GBIvRCkD7=8k zs2_Ijjeol@GUA<`{m!JCmOZ|hJ?uSCK?B0=#Q0%;FI}^3Nmkk?Dh`_*MPx_}bkyc# z1Oit$DBj@|G=2-h;7%%qJ|2lQ94$;p#9_-mx9kn0XU^?XY#(iPuB|P4GKFT}t^vg+ z>QGB;IEW@3#WvNK+QGs7~YAou^#0q)?JqFbaXVxj_|7* zKYi)foYBaLjSWz=k?87`ONs#PwKVn~XA5V%$?kUKt~Byn7B-_1s?GQswVK)W{x^FY zpOML`rRkSgHooFtWdHu>%cV4jYqg5Z{B2vTDznM@*=M<;n@G7&4~;wm8a+Y;aSI{; zcw{WD7Zd^{o_{KW7dh4Vr+0h>-`nqvz)gA?ZwO%;xLb&v#nLL)xmIIPIHO|BdV z2Q0L3@Lt@gcdLhEVdPDU^YPIP>%b?JT2fXpJZ?hbI}a?dd_tJSjT$&1Cm;)7NDj4G zbId6vBoSU-0R{8m%Dh-iFx3!JWv{dz?Y5QXJ{=8wnV+(fbzTxC_3t*Sm3#R%c_v&HC`bC>}`55O5!^Fv| ze004~a#d1NP2yjlC+G%48VDe{Ad@dz8o`r9N4WXS0BnDt2eNETrz@V~c25oS%HSc$ zxDud6K0sosT2Mbi3<=fv{wE`3iX|&|oy~#f$aEbz6QyA|L)7Rg-8#lBH2OU|w1>37 z&B*X5UHHMMEeUPhi*3jpX@2i!yzp09`i!=!_!-fh-fMTmmsQvYo#JobEZ@vFd{$x? z{$!J89^o@CA-I~%7y9vWf&oL=2xNnfr|akGR{U|~4*~0%1wOa&?_xBz(fCQOuL>vY zeu%L~$}$fTI_&UsC&`*lRAK4Hmjm^k#-TyY_^0cAZD^|1mYL4%OIK!-uoFnER=U0j z|BUBf+gNewuFF&vwdp@ROmCvM#$DN(SFrI#VK*YVul!L{D_6lm3imaE0`^i?=d^eTu+n6t$B?Q+cUC*ge=;KNQLy)7w!?^G( zm(IjF2W(xJc%7A5{@+^~f1kRUY&GLswci^SRQFY%{SLZNL8VjfLR^4rjF)h(jEo0? z$>Up)M^lx#yd^pAR!>fx{ve+gw?( zbNWii=zvjuX`ni(ad^*u!LrL7Ob5R-IvC4sl9u9Y1@*!1XwO(*d{fhS8c4beMrKFv z`pCXZL&|ckk?6EAd*rnpT6T&jw!7hN&N&s18yiN)q3vPKmmbr|TfKHrggfSbC@G`% zy!Cg&UN3DP=~Q$LxzGt}73n|LN4TQM9U`^(8gubTz>&!Gm3sAKuC*#%BGX1m%Dh|7 z3M@K}#8Li9@%wdUiGxewM?y&XaKr4IqfUhh6})#X-#p3B(yiXfpId5qpLwWC;!`*^ z5|42qGu5(m4w%gKa(mk<%dF!zfe~|L{IZ=rntey(LTIP}!S`zGv{Q z1Mn$^0>!N8+4WorcO=b;F$13|$`%^3G zIlC{H!&QbB3UgO#{ojY^*B-BQUzi&|+c2o{WDix)VDxEn1}SJ5GM_{1rf&YIRW8MO zyySVz`!ZQbddZo!1GlJ5IrGh*t1@B4XII1n4?SmUsU+&eK0{jGzHCs?&5{?{+I)>udC|x|XwzgmldZT8Nac zuHFT_!FJLl(AIA&JW#PhkS%}`N|UD3-=$VNNb-&B?ta~Vt{TZ(Dg7MU)j}8^nR_$~ zDYTRk(4dzcMC}Fk(sfj1U{!R$mAOS#7x&}!`GW6W(Nlk`slQdP(Z}{kK`G~ zX~!@X1;(hyh*40EP*G4(<0djDCZwb$%w1qrrajDgyFJWxbOJ$5pz?Q=0PUaqz_3m>OtY-tNr zWTh+9kkD|$!}}MQ`B9XE$KTMXb7{sWoq2oMg>-OwdvtZy*F}IJlpX;rG$>-g@bG}$ zW~aervu2yJ+-qTh*%AKe7#)X(hKi0~Q$a&TM@ehdQE1Eap;f7Lwn^{e?CRp?{2Chr z2Mh20@Z@YQFf`}gjC(TtJ~1LP%_y zDqBsG?rhO3)47+$`2z_t&Xs`y)CUk;$RCzH?4S!+CRCU=GJ|6aqhMYm(Wt780G7JR zF2oGWysIiQy_AfsZ57RZt{}4QJ2vA>>;?ox7(ozN6`4Z#ewcOpM<|*%8Y)r8hQ)}+ z#gT@^28Obgboecjs1vLTkMR8~NChw!4f~V=hC6vLu%>!tM&N#gL5M(mLCS7d5{Gzh zyLj#a>e|3`w|q)?4H7A8DAs9oQbb#QojLTDJZ@DAVH)p)a;hlQ>Oe=qpqm2YDstEZ zCgusI_9@3rsuoBzj$9%=+sK})s?mcq{Y1N_!@COYQF%J7>x+PK3##MG`mvU?p(hk0 zWIAAv#9;jcgAD*ctaRN-net?xz(1Sx-U+Dv}QfT=$~gE`;{F znEMD&nZ{J_g!m|c`ZV-1uO@^@e_3u1k@e~r=4C|4 zC7A>5_8PQ%s6)qBAEDQ*i9r2s3PTSc7i>Qe6l9mXe1uI<&5X;}n*X9_C*{Kc)IVz< zsnl~m*=OmZYd&*>ls|@RP};-vDUF5D3*EYUre&*+iCIt+pIz^Q#!n=GuqYG@Fj3CQ z<2?))g~LEJU4xl|L4fyX0TPuBv`i9#2hB01U~8nC-AG8!J_;HzuK@FSC8R|#Y%q3H zMJj*wgi@4{k`_KK0r>11rT_Yvueu`eVG32zet?b!T`ni5Bo|0458o7MjlX!gC`le? zr)^)EhA5}K6v@yDBgBMaQZ4$8QYeB3gaenkayOe<|2;Zgv0mdmOW!D)j#7bfW z&;cj)3Jiv>}1c@Mi$6*B^0Jv&=C7LC;G|&w~9>3-@X=>bMqicO@h`v;LLI zOn~n9SR8z?XSO5jmh&bY9k#qib2fV#md~D%TVZtamI80FuZGyh)!A>%J1x-@_$!AT zMpr|x)i<<@qE$@8o)&zP5WFA`{#0oQNRl+EYi8_WrV&Ar*Di!*0;`&Sz{{f*)2oXP zE=4~ECxJgMM~&3495#ZH50j!?4p6QrC&1f5ikF@f0C>0Ev|cBJQ`eR03`{KN9BN<2 zs2lk$g&~WpTL$=NVIY~xKkTBv6c8p7CtWIbPSZa|=0nLr+)p|gTdj2rMY&s`;WHd= z=*_{s6fzosFx2g$r_Je*_QaCa;Zr!9D_I7*R=N~yh@OJC&;a(|VcZ)iz}m$hlEh(p zjsSZhVe<`HJHGAae`)5{{D%X107mIX9UOuZ79dXAE$&GwQ9zUQ13y{M)NOiJ7l$tu zd0JpuUQwcBIMYhTZzf<&h>Dbh?IuYu$83mn#W*7>UZ6Vo=`u)t=6g+n9c2_*RO)Zl z34ycZirD70$-?}5#bGQ;;-F7d5l zhqZ|~m!6>Y_19%wZZb-k_?8*)icM7Fh`aCSwbW<3ZBWqKqi=P21Tc z6LY!PGEChTSmtNAelTLTCS?*kQk_jpq}7X3sj)XY*4n|>d#LHfE(yH$2Rp-Sf{TM4 zqC>zz*g4%k2C#SmTn=_`XftGw;^TqS(#x9PDhprmrP}!{4Y>WQjvf{S_=oEkZRcp^ z6lzx{rp0wS6=&BjI?bFU0NrL+_NwidbCIx`EoXiTtXH~$4N$I5?aYN;)VSxly{U+t zCx6UW0rw=M(JoSfL;)jp=sWd});m(iN^!x&<+}N{{F36oP;!V$McRdRXsVAuNTLCz z?eg$p#Wn*gg^d3(Syh3 zoja>D_#RWivt`TKV6yoXoF25EMp${#mY|uajhYWrEiryzo@ZOH!f?&=mkJ8tzvy+K zW4DaE*5;g@=q&Dahu{_zdK5}wzT{MGUpZihB#Y;`yOjpRsd?miB$gzBzUF5Vy!>5Z z$Ij3IIDWJdgg_XeWtn~3d18;Lw6V07t|=D-{-$)oHsdGfMkFt zt`E&b*zCqZL(7w^QzZ8X$%MZ|%zf7Gw{e0~s}$}oPH%cMLkdroGf73>ipPMc%IM17 zT5^n(DUpk_1``POH|yUy*FlxqV?^&JvwPg z5zya_#A+`Dy_>UmwOWC-i=Cm$3mAK%AbPb%{auJXWZp#Xv_^0DnYe~2wX1YSlM}&S z4;*fi7yI?8aacWEy~qTsp^jdZQmrgZfpX&`rYCiECuc4Q_`5M-h_#GPjXIdKKb3y( zWZ}g#!hH7VEKGX;I+&jX?|%?cP5FrqDf9^$q?Mf$5=F)}b~(a06Mw$B_c zYmM$+=l+QN9U^q6_d-ac&H13Q(wmgSf3A?|d@wzE`N+hjSJb9dOJyVRK1&t=>vg6@ zN~7;qWnwa#EVDvX0zfCt;(gdBYP*O#PhL4hmdy|oal!fJxWn^stN)st4{)wT&og?> zrH=wT62Do0P`5psFGa{qSg6#5-dzpWQvs^P8p!;lub3g#JMF{}P9BAZtmI;mRS;1~l+oLjcuYg^R7k9t%BrBpHqWSAK!uoB^=UbWJ3{FClC)4c`3dwfw6d zKq4@NoXS~;a#mip@x02z{s(u4P(%N4Xo9VA24F{VYbtAbaUFfh(p646dnY?G41>MM z&bxL3-v#Ix$BE}+W8}^u-hqtxj7nNHHilZDk-zUJd4dM?7@h&+ocb;G>|PU;n?Cn! zTSaz1$x;(`p6D4pvQ?h#w_`ttN-kM<>a~TOAAj;oU0sUqo;rM*ajepOV>*~uXcIop z=yc7Ov|Dnd!oqhJ?+w>S1{L7B=wIo-i@48ttoeMWRehXh;zpu@bHCN5oeDx^gA(B( zOSpYcB?z=2LC||FzY^x;d8E3Lah_>s-9*)HD4rKqtyp7$Us?}he)+Op^)(;YsSIVG zqglmEAeZ&l;L>7|&H`-mF47Gkmoo2yy>3;c)SlYU^I|J5k;e*x>LyW@ZF;kAMw0QD zNat*=XMYY}@pbphZaD>w1}C+B^7+Ekax=ZyPpF$idqp=Rxt`0(_H$Z6Jfrc#H6O#_ zqtKII`FM9ajuuvzg6qt-7x{Y^-yX8KqIn;Bm@U133(xmXCf^PXA&VG`TY*(c{cCn< z69u@{r1Ic;gYM$b{VXitD;`ySWybK{)8w1)7Gv^*dM!!jMx%nXe2~igI)0TTTfsIi zvf|IiGVPkN@aF~fh%b9Za&j+xX#Mfbc!i3RdD^r{{LA#;-WdJ>VXlDggnM@VBKlda z9sef5b-<~0{r%|zky_~FJ3jL|n7Z8x-qx#q_>Q?I+Y!HMXC?tjJ;PF3`-e^}5*wwk zBm83l`jr{0C!e17!CJyNR1m+x3yUni>63r+3su_|iL>x^{Qmf3*Q$jyH866eg`N%D zLGSNPe8SwS@{VK8@8nu7r9H0Wr59bRW!b9%!c)JA^{b^Lbllt&>K@8{8Zcs{SYx_f z&gjeZFlXK0QJ5Cg8-r^RrHJR*6WC>W@E8#7VTy|5uSJy<<$R{?bQmMO4m_`U4^?og zBHK;UND=_jfz87C?oc+JUu*KY^H>nm2x&MRygSh0-Fx<$T%~}{hM_KMbbvj=a2%1f z)HHXft|w*R+bcb29v&&t%G8PfkKoy(Po^~WK+Vy9Lw~Qi0D0lvqQQodYdtT)z@Kj{ zAd?c_=JYM*;!cE-wP1qG0wNK}vhEDR6QpI!yt_G2$o zKu1LjegC0R?N7dvw@EGc{a3=cifD8JxbIb+$9&5};P}y~DS_Fl1rWbm-rM9eEY|v8 z{@&4#KcKQQUOemem+O+7pJOA1yo)CxpSjk&CoLU2`}eL=i8>hjVh^icZo7!g;;J91 z{8K*uf~|T}etJ`N+BJUM&jUDaQ~+Hc_SNQlLhP*rq4tpPbz>T{@UnKbqNUqUgD;gZ zsjpU_s7B*3pF3QWGpt^`JWRbR(dV7KdftA6{E?=@Qu&Oh`xxBIT6yU*RcigEpb-F)(T3!=(TdHm1;3EgEC$xWC0&D_|qN%I+ zi2W7Qjzy1>UN`Cj(RfEi{&~GLEH2sBOk*^jno`=t&LB9-(}zLiN0Znk#HT#3 zvhmQORFxupVYsQG7-p*qB zWbm2mZc%fj?p1A;D;v>Piqto2Bp(;enl4lxpT}FXFFGhi<5hjW^wvvOyXdMugm$0s zicW!d1NqufeN!?o#sx2z6P=L62mWH@fFIr<@YtpuqyBhoyYJq&+C`JhJ*B@SV!VL= zhP0A=HrH-@OAjr1%#M)rS?jiVL$|v89`81EsX5;A?*5D#ba@>qj}dfk2oH{Jcb}rS zM!2-vHrj*9Nk)01ii_Xnw&f!`cm}K3qa@B!9An-+i=-EDX{*w{t%IL^I2GD%Z0tYa zkYXFez-CIcRJ)Zsx+t;us54u;q*KweOigNId?VSR|M7aE`|Eo>YU93Jslru*bhg^} zQXHsPVW$G0RqjEe-FCj#o(=h0>RXOhaYOOjWH2~{zleoAYt-EUqlE$cS)qb05i`n5 zIryEJ-HaApPEoRknK|nP2If@wxQn#IBi-NK?>>F&kPxVf^@lL_{nlxa>O2SH8q^Xr z!VKxvPntktt4F(_kjj0j`RtRM$#HlbwcRFK9n5`DBEDP5Xq;nu+Ztlb^s3fEa- zs$-z=DT3gE8pE;l@zh?YIMfU33;AEK=N zB0%2?UKABC48%JceI7_$InWQse$?0!gs7zjK1Ed$SEOeC|Xnt726bsWDFeP}3 zIzYwzTYj&+44L?S{RI}#KJ{^(Y`r*9MKYfFa9tMxujz7*`gN$tQF-+}ctdpMM5;ag zsol&m2QK&*ZI*!Xj@OlR;_%=+bc`G&ud7G#Ev@-2Z-KUcT}bE@J7$db4IQxNKK2TRb6T9*&9a;13nXAmilbZnrxr zmS0XJ3h(m*$#EMIrOtsD5nVuVmiK>Kr)KQgGn`2-{$sG6^;=cPfFXV3JX3Ucg6O; zcJElNHec^vzwhLB^35V=zxnFCqp~i4T34i+=PnZXQ4dkxk>4&>fMvB6m54OinB=nO zf&2;gD4BtJ?`tM}SbMp^ykE1I{p=e~#1q;N-)6cPAmqO3iR|PKGo#bpmv#RbNtphv zSvD&=v!L3c>tEvkkwz7j8F&|C=W<`u#$xyH2xf9qXSwuL0FEx)vn^gk4kZW7V9z-p z&eh6~E>Ee*AAa%bl4(MAb~~`GtsLnCNYn0tLuy0O^K;7Z2Fx+8!L^Bm73QUV`@r)e z!y!0@k?0KTJbi*$^9^w3vW+2)192Wn?LA@sJT^Oa5l9A%pdgbg-ll>wFe*eyfKN{pxbZlp(uzk13{O=r$UfOP z$}L?6;_jJ#Q~D0M^5j*z@`+2S{a6P*6Bl=Ov>jD-_m^c<6?+MMhu$2s01McW2zzpK z#{gPDd#n|Agoq7qJ{Pa9dluAUbZ@&4otIE3M9CT_cj~mPNk-p~NxPZpLBK{D-GF~Q zE2Lj9-npBPD5q?QGoFvxtk6rJ6o~1F@z8T~%T^=1U*5*=CZV1X`_dDWX?2P*((|ejt z$&dJUtC8o5*I-SJ_>p8khQ#mu0KGrM#R7p^lpeK zgf+m_68P~i0Yw(f5+u_Y)Dg6FomEq@m_i4j#Pjh7EuzBvxAtA$@;6FlUleoGyO_Ixp4gG8G7Lt{Z>#gKBP2w z#LTLrA6X%ReE5Z@v5g*Pg>bX9YF-8Z14cl(zgPa-I(|{9pHHV(_mG@)Vq8y4xE7Hh zmDdbtdK%r918Nmdh=nxc`7q?)SEx0u2PKCbSOcpVWEg?FQtox9uhL`1 zx%7w~cdm5GEIU=BHL25HfAS%6im(!g>8Z6zhzMd9Xp_}b307Sjpg!&>+L%ZV5vaQW z0rnRq9XmO0b@J337b<3hHBXvaQD-KBsMW|iYrU?_%}dj@633f3blJ|B%5DNC6tB1e75{xU+A?T&v%-*->IG7 z80b&<&jZm*6V?XJ6bvXoVGI8K`?Oi8WMfz17hQX8Kz5Wn@a)n?uNq}`3|iIQ$|m7i z=Dvr#tHzC7P7;y{h-}1KmT+M?(mo}!_WF;^^&F&o^fMgfrd-4NI!a^kl1mDC` z`%{jBlGcZqyHk&|?R-gGO-}V*PlCH$>wLY~8cd*ycq~#pc-j0F&F}??*dE}`VSI1E zy^XYcP(0Rr0lL!XR?p71t~l2|%5B!qJN<$4zAeo#IPie=T=2PDj$wn+jn?j=Qq;uS z@utU?WyN;hHDk{YUhmfo9l20@ICi&Axx*dvVYtH`VooFv(|d7%XoO{64(jNcR>>G8 z8&=5}CL7-Fsk~JkY3F}co!9S9=+o9j9)ZKJO-b;KMK(e_7 zkUvx_+%z748-O=AA0i&lEqK=k2kTHxmUdzv<}-jNc0Ml!b8{0QXz5{_?p%Dujp2$c z1;UN?WhZS3%0c~;6Bk=JEetN0gE{eTw%y}{+nPqV_A0~tUXiVyAmfK6GB{b!AReK%V^sLQr(C^Lk&3`_U z)2^Jp{h`E6Kt4N^W3Oeg+Y*}S-L)D+Ha3#aEEP;5x^?HztPZDkM1Nae;iLCkU=1t1 zF8RV3^p-kBvif+|snN-pZAy78xe^RGe;@LB$4OAdRAyD>7S;XqP}*)!G$rp}*XkPr zUE6c7^n-QxB#xDWT~g()pA-#5rI7(hQmYkH5Z-AIdCxRuBh z=RP=_KS@fBXrfSg)ZVk|^2ac3D`k>qWrr!06pY z?@f|Jc29yxWPEg+un$v7wA*I5@xC5@**9s$+!~4)&cN~{W(m&4wRl^zcO+jVU!{@t7*E* zoL=?3bs0+S=D(TH2L>j{PXjC~1icbxgBvN7ERkDehfn2ERDUr#fz?R%zifi+Tc-p+IKm4|FGKRb6rK(xK-5Ru~zYPx< zyvHSwFw02>>jIa=MF?k>4v$OkzUn3Asm^4_lqF%Oe$~D&;}367!1bPY*_mGDx;}Ir zWoiL*I{zyz|1_45?f?%;9~tG(R&uNV;e|2=wZi1Fmj zQ@TC_sX}~nM<2q9gLfq|$z;u3o#27|P^D}`Md9G~Waoa&>m(f9l`YTPIXq4;Y%#Ut zjoFYP6>~b~iy$D~`TWTQN?kwQ?xWT36?EI(_b))5QPdiF)CI^fBxave=K?gLe}K&R zx}UH2xUwPNu^=OTPJl!GNlHuJ5*faEScYnRjBZC9jzt}+43pYWnom!{d0Cv&k@p-v zl4w3IQG($#G_*IxT#P0ytlY&8b}A15%<{d)QWOHcEdYsFZl#l@Q89eRWOJ0YpT^dN zYtidT*v4*t#n0|Dts3KvAE9}b1k2ZV1}fK1a!3Nh-=LWR{TCoORJiQq-G)(?EJMg9 zd#Pq?`TR5`^H6+T$eX#LW@n_Mz~3ljFQGYo%ntR>x+YuF^NHarrtuKx{;Cpz3ic3U zR^sp#6WE3mOl3?CJ$)Fnj9OttnRX$pqN_tw(qCt7n=o%ap3UP@W|vRE z_qoh3>RGrwTZG(H4yGrPuHM=?(4@6FcfbiykBfH>Da5qJWpxUzZui$*-QKFTwOT_L zxsdHoZ|BMxZ$8npM)FCKb@ayfZh)7H2%lCz@iiW>)6`gwW*(<4UVyHK2@U`mh)%I5QW}xe->YmMJ3Uycl2GeGJPoOG3+-{c<~luq`OydTM+_0AoJ` zmzuuza=0!_XN?1^%&)--4ee$aM$y(_2Yv{3CYp5aU2^?;09sJFat;M|(fw%HEuXHl z8TzI{A_qcFAS*WA7LGenm)NzSC98mpI2VX1+Fu_f|oKaK2va8v4yz~$o71i9BAuiZ_Lg1==fkhsfTq9WO5 zj|HeQGhbQc0iHffM9p~!$mDN2CWc0(;8fyG z#r-D>vK>Cy{|haq;V1oFSl6hb+Z~@QU()W6X?8eY{b~FzHKKVWgYy$cMJMk0V6}SU zI-8UlB>q;Am@l$n5PTL1Aj;B=dH_Z*{SfK5x*?Xf5_xus{4*p;cAv>K?;Ic=#^ejo z<&{22G}}>|1t(3*eNE4nqjl^Bs0!@b996cNM&oxFCB}P77!0D^kZdyz*Z)ll|z1=LDn<}GdSaV|vIBUAMpanZo49Jw*iTm&zxz!5TieI0zo*f~ zPx6}C8}yTvlIzR@kS%A;HwB0d%Zh}EA!oMmx9om0N$=f)zAPtRkK*p^d8Xts%&qj- zy36-|CN}A*(A0+Nh9`s-&(UT528HozU;aluAnH0Y>H>6qzaUz3UX8u?lmCO7im3cH z53bni_y9Uj7aD~Wi@JTjt?VBai!QJIpaq>r84CX8DY3zdb-(HG0VY9(p2s*v(O<|p zB!aCY@*4eq2WxtTr?=pn*?3h7AW~{d|sb6PDE4kZF zP;0--A9|XwF*Z|J%bDZj9COy6BWDOtoN#&~U5%z+7jU&*YYG}fcNU+$=B!y9>nTx2 z=w^gKc9Hm78tJSLt;<*3b)9Umt(B6ULZ8WY%H5oO!tbP=fBGW)ku7qY3!8oSW^eV= zKXkaCf^1m5NhU!G{DwQ#5ZWe_j22oMnAC}P(`tLU@BixviD>p|?{MDpQ*=^S3cbQ0 zU+DKN`SH_7SLn`1g~_oOpv{`E%nL53)EdQI-5HnDVJiFT(vC&fAs5^x%I25{Tu66v zov-|MNjbf38Qep%IweUW5w6jl4uFxiKgAA6Ot8G_w0K@FjeSE@7w7+Wsxm)>XI^;- z@;*#ljYohI|17J~O9GX&LY}F$P^4XCKfrXii5MB3@M<-Eeq{uYS68wuQyx^LOBDT( z?bMD?bXZoRiCtZgsrV~kvXfgaB8(awFn#oN6S?yalEE|(H}w9XZpAe$VJB$|*^Q~3 z&A7K=sZ_0GR4=KdtX7<3`3YmFlkj{nPYw-!{kCbX+EFj)bMbKfx{mTy&YU|*vq}{M zH7Ro`sa(1iu4H7!@{0778}1*}e0vPzs+x{5%Q}rsHhY4WvArw} z8!0EP`RCHN(G=DKq#u?pT<;9J2wQG4k4P-%H|V;aZynsi%e-bB>q7J1Bf`z-VB^^0 zH0yaxoAF8-j{0@U?KQ<%HZ$vK93|$}N6zvy=xGSe=$FbS*{Y(xP%HP(FIZT0My#F)fEh8{JjbAK1$zQHxh zi6!8wXk%%uaI0h^s6M$=)zu~#%S^dik@9@Yex~FOm7i?%qeckQ=1b}0H%fcC3&+WW z9luo98T|6ofX;_eeL}08Z=V<-J%MlNtbW@8rZN5Ha7}5)zO|U(ye#omvuFTUx3;b zvF)hfPx)L6W3KA)t6P$z z*^^$=?|H!CtU=90TD^TTLt(EuBofcwX--8syzrFGy+gV6e)m$dg7>^SBD3!BIJ4uE z$!L*^y|#n9hreu+6K#^_m^1C-^eLTJyQ+FCY9*^uHcqusCL*+^P`hJ>_X1>BaF3vs zF#6>JRQkFihS?ymT;-G2WXai8BV0$W66rd&}dUegr1 z3b;|bjugkGt=P6%Qr{wJ?bJw_Ue9gYT=<8Qo6QCr&(Ng8ccDhXnUdZ*xvgCPj$q-+ za8ZerrP2U7b8j1|rPBX?w3I>b$3`FFZGI0`=-YqO@)5-xSQ`?Dg*5hT6U;s3@{LiI ziJ{Sz@K^0Ac0m6pr1?k4Y5*O%21;WGmilxg>H)A>C>1L8^U_ZmO_SW$%Y|2BfCbOD zj1LO6W9T#gAUp9U<0qVoWGmYsYxV!2tab)nUa z_@*H50a8XcS?&Vl(C9GYu`TQbFSu6XI-{F+9vBj$rV&#ot6Y5Ya$#oHY$X0hJ6pd0 z;dcpe)P$wIXZ^B9f?xR@p>MNQR8{Z-WHs6$O$y6iLIoy`jM)iHuvF`%n&21wx9woo zx=q_tPtTA=0mMc}!ITmC->@TR1F`37X=Vg8*his+5`B;a3$@oEU$2Sj|!F{3yrkCc7wbM2W)?;&R$L@vtz zpgC?Z{^3yF9B1F#Z15rZsTEKmUGmjECP4 z2SUDdm{U&j3;-m&Qe$EyfbsT<p2YXf+M81KF;*zoQ~azXJuNF zPLAb)e;CzjPq8hXND}G=e6c!^X$v>2i1FP(;pQF~Z5(1k5{bMT>BcYTx7epE^+a zJ?-$h*s+n_JyL{pa%L3lC-o$_m*vOAXFUmqTg>%~*dK~7n}Z%A%5SoH_#(C%iHAVW znX&wm&B&~_Aw*TlPtV`mrGH!9cH8BZ^i{TsiiAh0OCf)M5PR!m4-PJ`T^;A)Kg}#{ zx4Nf|^gB#!RuY6*?Mph43zps}skbnVanR=^Jlwlgayt5{{@Ps+hWk3%$k!C~&lkSk zYoYQ+Gb?MJIHs*FO`GUS^GypgPCBjlyqvd&T!F#WaW)cHynjS?r7&bP@gbs|*&S9oHAJ*EPB|wiO5l_HLo3R)I%;J2SJ^0IiWcPi1*i_T8e3o12jOHt z88f;7g+R_3IRY}QC^--`HF*HtaRK5)ign-11ve$=4vTW3hx|}nExf1ND;JYWdd|*U2t3=G6t_N0jJ{w%sG%kq0>u|tMNMrY>vWz(= zeB82JG8U@YS@++=`aUJ*DOhutoVYO59APBD3Q#sIiPj8qdYx*t?7-lBJHht?bOV{C zxVycTvnD;y0ozZA@-SCjc*e7dxRz&i|LA#>_O!;W%j z=4W*5IxMG=@#c`C&|iQOGRcnv&Vl5kEFvO?%Jb(lz0Jd&1@Z@8shaTM2>3H^FWUD+ z-TlHu)r@;QXv6|a&eWpchWi>*zI2j=ozu-f$&go%htm1jG7i3g?JP zbu9XCc8%uz+gZP_zUA6?BjW*ag-~sX-0xp<1;S*G_Fb1i!JHF zAUacL6@k_;Cvbl}tHbYJkY326Ul0;v#BTR18yM1`>-NC1H1?*(qJmw^z^<{603uH! zdSqx<8BmnhkQX3p#tTs1%QAC7Pq`+XAWb>c15*#q>EAD`#StQ;CT|%_Ed8M2{S3!uDGm)w3!#(f>4Y*kM&gvAEVMvkt1cIOLZvh@nq0{^{kSnXWZU zPS^OHd-{2sGx!To+Tr9kH9^XdlBers)utMX)TvR-v)HS*1*W$1Z_#~O`Rens;?D8_ zYvXASR!yTHtaP&al#M2`T;xt_7>krvK>E0?!?Xj<$a^b?cWvj+q0Y@SRKjQx&sbGf zRA2H+Lmc;mp5%KSs*x!Cy$-0=oETkUGcLVM^M)K_f0olX>kpSMPw^a*XjtT=+tU|0 z@1`v7DKV4J2wg@(UkV3*BXYuG#qT-3!l#ZSdmk`tLG)tTc7GGG`~!lD$My50SbkBL z#w_&pO+4wGTR)!o_{+>mcOB20pyCMDt@i&`jd=A-*=vFbr3q#aaYEHuxogzO`6kb{ zlTJ8ftB4w{aqLN?cI&Q^tNJ>SoX9a17omkaoY7Z3t>FJvd@1>I)9sD#mBf=pNZ0c? z#^2Im`m*mI1mydC%bKNgN6nh26374Y<7ECd%tO~ze+3KEx38&l{oVre2o0cv>_yk!IanPETs?Ej%>AqJ|d`C`9J=c@pA5M!Q1s>YY9$2Np5GC~dch zm1;pn&*m1XaV2QS3(!%4Kl$A;78ona&cevFbr&^a&*0b{Q?=4e8CVV*kXyyySY*CC zX@IQoIr~)Zb^oLUXd@{eIY!>H3p^c=yR&Zsi>PmEOtsv@A_fi|I5a{i^1V3JC^HB> z2kB`XdC1fEe`q?(fF|EB?&A*>6$=rhQ930@htl0JX<>wvGzbVoC1m7)4VZKzjFAE& zph&007@bn1yT*KO|IhQ>FZSwwu@~35&N<)n`JOu@vmicy@sfOxrX>A*obXlddjs?b zE;-F+vl2RcW$)gX8?pSh@67ry36pWV7Z9O_-*{R>6>zG0JZoQ_$iU76`X}?@^hf`7 z9Hmvo!slXslQU!lAAXPzB##`6(MBnn$q@7oS9|Hu%D5 zJFPWnc}tGj_xRQ_7_<5P+_s9J6^FgOG}j4k4+yA?eXb5;Jm$_P$u8|V8I%D}E(i_X zjf#`s_O95h{MbqipVT;CnCEx7W9u#sU z!cQ$re>yO|ms+@B&L=yY*b?)^W=b|;daqcHiE1X|e<}u4LH&SzioU>5L{<%L@mkt- z&ME`mZa^A|xa1}lSFV1mOw(Ax^Ln1Q^tB#0dd7TSQTu3P-52Nw8tB{2auSgVPR^1e z&)MSJ*)WWgEP3&|R_-+iwY{-O@c2woDWb5UXY)syex=WB*75?)>|z}IMG;h->jKxl z>M{Y##wiV=_tg(+ZG==qjG??2s&!5ieh{s9zcy7O)6N*Km<3PC4=Q-jx~liELu_=^ zGvoZ7DuH+H`_oO8{`JR7K+X*iiv}WEgOpzJDMVe_F)=dvIZ(}-cP}$?KR7M|T&GkV z`q3jO_DtSk_H~zG{!AU0PxfKsC6NlCTJ3^x&id#fcEdE{dF$;Rpo~o_@OuI=FL3Jh zWq#msq+>(`x)H95J3;?9l+9gzOvM52FDzQZ@(!&_pK!n`wnZbRtFcoUoO$(zovZ2gl7`~76psW+WoqD!Clf%@qhEFxt1bM&8@X}x0WK4;%t$LLD0;pUqsZ1CfX zi5?9rMn5*YtJ*V5Kk}+FHTETE4X7->Osrb{G2&jw*DKMHq*zF~Ni=q+JBFsnnc8-P z*+nN$7q0K|@CB2Y@6Y=ru>4%rKca_pN^#);0I;d!;ho~P6O{Txs>-=SrKzAwBjHOW zFr?yJZGg#NNdULC-XZVpc5AukE&Sg-&ez$X1TQck`&7iD=P6kDi0;JVi1@g&zKeC& zj>F06*3&#w!WjJ@WU}i(h5%(NwR%cC?kCO3@J5{1ohA&=q+_1cTQnfxw4Pr9v(s_3KF@Pw&gek(R7j2y#(s zWk1w?tHmnRUkLgg8CCL=(R6&@VmhzujymN7UlURiaDT61M#i6=$;n4aRReEv9i@wf zwI+3s10G*bKsp|@<={1@X1hrsSs{EQebu-)?>CmEfi~bn8*Ec5bpRx~^(?cbztFAr z(!z8~*J{cqOhkP#=w2XM5Uxh+N&e%YWRDe(Ly0&JY=&xa>6}Y|j!%hYkCznf%v#P7 zi6?m_3AkSEzHWFRk80n=CDF<4A*@<-PIFxyD^h#tRm*~QradNq7GWCdvUMx_`CF@& zzJ)7JZFk{~y>7MElmcQS=3}RzGpwYk2h))OeT1IP0^gdx&9jvQ9_=#?@jIrsS!FRd zr4LG)ZoQDT*-2Qj1kseQz8bJR5Vn8jdW*LDdt|kv8~Q1rMFT(fB26fy1}Q4Zj1=`R zCX+;*Z#s*SIEG1Ej;qn`y}pJAY||(^4kf@A=@PCRS$Zou7flJo@If?m8;sU-Q7d%FU=IUFZhEXd{ z+6K0F?FW(sR9(em)EgPR;@e+c!a8!(t$FtdK__W&Hg*Yp`{_m&?uGw~S=v0MeSgR< zlNVo)MbZ?DV&wp^4tH^A#C^!t0}6R|srKzsKA%*1OC;(`!?B^H{Bp^VB<4$Qbn&MI zkOXuTcX1?DNK@;8A*)X(4Vq_dp~rIRGAywm zZwoXhx3P9hEgT+QgjTjBV}NzNwdonw`SXv_@Ai$S3VYP-M1&Td%Eo|)1?KFAe6O;a zW^4Fgk^<+`-vAS9;iR|9XJVSsLdUf}de7|HM}(m~yPu()F7MV{2d7y$d-%;s%L|^K zg3etLI8%+;T|M9k_9J{~*nUrQ((5OAVc`-a~tSil29!t;W@mlPAv zXR}AD0+IC6KB4ys<$kLC0feWICpqz}Q)FBEbPW48DbQuC!c+>e-GZPnbO6$KGT9sK zy?fte%zY7QMvs34j}7;Y`?NqJOssEl?H?%SSCP~~f3|@Vgd`8DS}7l7Kd{i2-$Lf9 zHG=J+0I$FUZAb}L7!W*gib_{H{bFr{TeTix7#QCE++Y|rP4@oc+Douyoqd87`WiM9 z))~LO4tq&d3?IS5gghL4Fnd;+9hrsgslwl_eLj>dlau(d)msBssUyhNP*W(NYRBQI z@Mb-SYvS2_yIz3>M{_#Mn~aC$s>)hdTR|sG-?AnokPApw%fmD9(j_!~)EdJXQDM)g&g}~=KV~6)JwGu8&Xnxf9%t4wwprBs#h&dQa{XPW z1MFhJ`Xxbl`WT8+`W6eQ3~SGacF%GDsh-N3t|~1jFkRP5;GJ?jSo9fF49g-fJx_Bg z+6e+rwkb@$<*W0r3&jAse#;9w=K8`7ddCE}SEE~tFmbC2{~c%YOYK09&KBLToL^Zj zkL{-k&5xC|Y<$|^=`6}=sWvrGnhnWkRlG~jGkQX~7B|sjc;gxuBW`NpkXJJ8nKQ?; z(hW>1l#i+M3kv50N80RUT*8i1bfjPv$A#}jxbAc9EYcxbOqh%7=+(^Jz#zm>Ufl?i znQrVMrwnWDTZqEm!?FvwP4*%NOnKv?c7EO;5(zH7zM8;3-w|S!P@)|@k22*EX&3m! z#Gk!l0@xwvEi|1WN|NR!~wm^Z2<8H#2*R%m)2462U4Z%(ZJ4zWOXH< z=O4wcjo$A5!^rcV^*5k@|63ceVH}pp3uoUje=6bt+#`vmC6Q5%_0e~4>i7cndafsQ z#Nz4))X3_ng$m_rNREcnNW5bV)iDTBwz)KH5WS9;*u1z$;bZp<(LMMnuiq~jnB3Eq zPdd{qgOZtIgi!+#;my@w$I9o%FQ~I-FY=u*C2Wamc2bW@$1>@7M*Uyw@~jW4vP7Nu zn`Z=gSzCVh`g#=o;ls@hByIP}k5kTO4aZ|%Bi|M3034B=@DQTII#^uH!k@w-av+C) zjjI_miW`poh}w2rDv24Nv8zn)V|O*LT1NdriGp51e(qXUE2*3nCi?mlHp!_cuo}Q; zMRj-PGwqk+KWW4}-{bMl6n2n_mIT$ZdeGRpJoI$^nxeC0yMa@1+NaK6yM37l*;0KD zkcbG^>RPG21FLcX@<6ny5AS0xlxd6v1s=slO}@0x9-yi0YwppM!>s^N+7Op;#6saj zLfgoj+QzUfgI@``v_Mventyk5mjo)oi!5n1=Y>_ila*6I3dn^$X(>P@YhVGGL&7?RS)H-6S(=H`zuEzRnoc zo@sNb@GHrpeWIbIp8`+CB1MRwxBG!Zr`R@seX6s~6dIzy6sn!x+uN1AA))7hc0LtO zPJ6@V6TcwTO;A-^wgFqC%2Yi=j0o>#*w)EICo%8U@!FrdUu>H68XN}1*b6M7-IDY! zLVo-u=|7k-H|%X3T(AJO;@&M#1BikQ(gJ~eqY@Rub1qh-Tt zeq_lUN}4CsY%$+Z(N`jsPV;9 ze@Rv@vK(4r7c$Hj_M#dQTyiCqJ65V~+$R`mS0ldnF01g*Y>wsZ_T5ZnG>*MHuHufp z2lSCIfMl!AI2N7QSV~DnuM7twMF((e_)){Vr2x z#$0JJ0xI%!X#c)V&=L3eC?d@`q8E8w8_)B6^s5{n%JCUz#@&&|f{0$LxH6hZxUF@~ z>!~LPUW(D9598XKUEeCNzmI`(-Du)b-gw{dB2xP%`3e89bT7${ab(wqbsTpo*JGUC z_i>y=9m9JshbKynwRO(NmSRfCfu?K`!7E`*aM8jp zcSz^l_3B;1bw1JZ_?=C?K%Fw~`PfMP7oA>&fk531K8~T^4aS+AGt#K38#Us&vXxfl ziVpf4E2RukN*T$cY=;%vrZi%PDS{rNzuT4^Od5uKlJ^k=)L_-!2-vQAc5ebp*VBcJ zi#rhc_9F~fFrN{`Wm=SqJr~?OQZ>O#*Q_KcRgaY0jrdNuElS-K8H#-j!6u9ml}PK} z9^?Bc4aQP!e)`Llb|}i0q-W)_X=O<)4;WnO;1DfJJvlz9SIh)r`KG!MEVsf zb1>iQpv8-4GaY>>WoGi>c`$AP4sYpleu8r9>TC2g5yk6@$cci}?U2kLwU&~q&v%G6 z>9ruaS4x7*SydB^X-l^aL(*%W0IYD z8yS4U(o`LFJA^%%S7Cqn2|?v;;PxBM+eveJd*(!c)`G99LmdLo#Y>A{=KduyVbDI? zv0NbC5ofu?H&nZb7fWNx>SqDO^^9raE(5&l@ltazbDKkO)YbIWUCvqxlttM9igk&m z!Z~r00p!!&1D7%7A4rrcPrvhiK}5udqb-f2rhFUeow` zPd?;l++PwZEri^58sN~_AcDHm1ZP#wiLnOenU5gJ*1^(tF~YC`6#GlffmA!+@`)|o z$lQpSo37$yKOH!M_wr{ zb(mQQKA-A!UTnpnXAV-11ih3rvmV{Mne}9}Jcd2>KeMQpISdoQRHrXa*y)cd&&&}Z zE!|zcGX-zyS6&ixtj*t2-3zoxWVuQEea_$sqY386wh`gftGj0mXr1{xHiK?SQ|sB% z^OOhPnhWj<$QvOqDu6PsR7%a!8*J+MLqCA)mE;9561d-1)oSz;t%o#cip=)9%6(NW z)!%oZam)lvIa;#|13|W1P7d{o?)|V)G2}>KWet0n(4A)07v$&Ta7|aFd7jC_P8