[Feat] Route tasks using linked communication context

packages/cloud-agents/evals/router/assertions/routing-assertions.ts

@@ -361,6 +361,31 @@ function doesNotRequestExternalLookup(output: string): AssertionResult {
   };
 }
 
+/** Asserts that underspecified task text requests the supplied external reference. */
+function requestsExpectedExternalLookup(
+  output: string,
+  context: { vars: Record<string, string> },
+): AssertionResult {
+  const json = extractJson(output);
+  if (!json) {
+    return { pass: false, score: 0, reason: 'Invalid JSON response' };
+  }
+
+  const expected = context.vars.expectedExternalReference;
+  const requestsLookup =
+    typeof expected === 'string' &&
+    json.needsExternalLookup === true &&
+    json.externalReference === expected;
+
+  return {
+    pass: requestsLookup,
+    score: requestsLookup ? 1 : 0,
+    reason: requestsLookup
+      ? `routing requested external context for ${expected}`
+      : `expected external lookup for ${String(expected)}, got needsExternalLookup=${String(json.needsExternalLookup)}, externalReference=${String(json.externalReference)}`,
+  };
+}
+
 export {
   isValidRoutingJson,
   hasValidWorkspaceValue,
@@ -372,6 +397,7 @@ export {
   hasValidKickoffMessage,
   requestedModelIdIsNull,
   doesNotRequestExternalLookup,
+  requestsExpectedExternalLookup,
   extractJson,
 };
 

packages/cloud-agents/evals/router/datasets/edge-cases.yaml

@@ -134,6 +134,25 @@
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
 
+- description: "Underspecified task with Slack thread permalink"
+  vars:
+    context: |
+      **Task Description**:
+      Look into this https://acme.slack.com/archives/C123/p1710000000000100?thread_ts=1710000000.000000
+
+      **Source**: Slack
+      **Channel**: #engineering
+
+      **Available Environments**:
+      - Web App: Customer-facing application (repos: acme/web)
+      - API: Backend services (repos: acme/api)
+    expectedExternalReference: "https://acme.slack.com/archives/C123/p1710000000000100?thread_ts=1710000000.000000"
+  assert:
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:requestsExpectedExternalLookup
+
 - description: "Task with emoji and informal language"
   vars:
     context: |