[COVAL-4591] Sync ChatGPT app submission tools

chatgpt-app-submission.json

@@ -235,6 +235,102 @@
         "open_world_justification": "Does not publish content or change public or third-party state.",
         "destructive_justification": "Cannot create, modify, run, or delete resources."
       }
+    },
+    "list_reports": {
+      "annotations": {
+        "readOnlyHint": true,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Only retrieves saved report summaries from the authenticated Coval organization without changing them.",
+        "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.",
+        "destructive_justification": "Does not delete, overwrite, or change any report."
+      }
+    },
+    "get_report": {
+      "annotations": {
+        "readOnlyHint": true,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Only retrieves one saved report and a bounded page of its results from the authenticated Coval organization.",
+        "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.",
+        "destructive_justification": "Does not delete, overwrite, or change the selected report."
+      }
+    },
+    "create_report": {
+      "annotations": {
+        "readOnlyHint": false,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Creates an organization-private saved report over explicitly identified evaluation runs.",
+        "open_world_justification": "Creates only a private report in the authenticated Coval organization and cannot publish it publicly.",
+        "destructive_justification": "Creates a new private report without deleting or overwriting an existing resource."
+      }
+    },
+    "list_run_templates": {
+      "annotations": {
+        "readOnlyHint": true,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Only retrieves reusable evaluation configurations from the authenticated Coval organization.",
+        "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.",
+        "destructive_justification": "Does not delete, overwrite, or change any run template."
+      }
+    },
+    "list_scheduled_runs": {
+      "annotations": {
+        "readOnlyHint": true,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Only retrieves recurring evaluation schedule summaries from the authenticated Coval organization.",
+        "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.",
+        "destructive_justification": "Does not delete, overwrite, or change any schedule."
+      }
+    },
+    "get_scheduled_run": {
+      "annotations": {
+        "readOnlyHint": true,
+        "openWorldHint": false,
+        "destructiveHint": false
+      },
+      "justifications": {
+        "read_only_justification": "Only retrieves one recurring evaluation schedule and a bounded page of its history.",
+        "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.",
+        "destructive_justification": "Does not delete, overwrite, or change the selected schedule."
+      }
+    },
+    "create_scheduled_run": {
+      "annotations": {
+        "readOnlyHint": false,
+        "openWorldHint": true,
+        "destructiveHint": true
+      },
+      "justifications": {
+        "read_only_justification": "Creates a recurring evaluation schedule from a selected run template in the authenticated Coval organization.",
+        "open_world_justification": "A schedule can later trigger evaluations that contact configured agents, phone numbers, webhooks, or other external endpoints.",
+        "destructive_justification": "Creates recurring configuration that can trigger future evaluation activity and costs."
+      }
+    },
+    "update_scheduled_run": {
+      "annotations": {
+        "readOnlyHint": false,
+        "openWorldHint": true,
+        "destructiveHint": true
+      },
+      "justifications": {
+        "read_only_justification": "Updates selected fields on one recurring evaluation schedule in the authenticated Coval organization.",
+        "open_world_justification": "Enabling or changing the timing of a schedule can trigger evaluations that contact configured external endpoints.",
+        "destructive_justification": "Overwrites selected schedule fields and can change future evaluation activity and costs."
+      }
     }
   },
   "test_cases": [
@@ -247,27 +343,27 @@
       "expected_output_url": null
     },
     {
-      "description": "Inspect the stable reviewer evaluation setup without changing it.",
-      "user_prompt": "Retrieve the test set with reviewer fixture ID <PORTAL_TEST_SET_ID> and list only the test cases in that test set. Also retrieve the metric with reviewer fixture ID <PORTAL_METRIC_ID> and the persona with reviewer fixture ID <PORTAL_PERSONA_ID>. Summarize the scenarios, expected behaviors, metric purpose, and persona behavior without changing anything.",
+      "description": "Inspect stable reviewer evaluation, report, and schedule resources without changing them.",
+      "user_prompt": "Retrieve the test set with reviewer fixture ID <PORTAL_TEST_SET_ID> and list only the test cases in that test set. Also retrieve the metric with reviewer fixture ID <PORTAL_METRIC_ID> and the persona with reviewer fixture ID <PORTAL_PERSONA_ID>. List saved reports and retrieve report <PORTAL_REPORT_ID> with page_size 20 and metric ID <PORTAL_METRIC_ID>. List run templates and recurring schedules, then retrieve schedule <PORTAL_SCHEDULE_ID> with history_size 20. Summarize the resources and all continuation signals without changing anything.",
       "file_attachment_urls": null,
-      "tools_triggered": "get_test_set, list_test_cases, get_metric, get_persona",
-      "expected_output": "Retrieves Directory Review Baseline with exactly two cases: safe, scoped mild-headache guidance with escalation signs; and concise routine-appointment preparation without inventing a booking. Identifies Directory Review: Helpful and Safe as checking helpful, safe, appropriately scoped replies, and Standard Customer as a cooperative, on-topic en-US persona that waits for the agent. Performs no writes and omits credentials and unrelated internal fields.",
+      "tools_triggered": "get_test_set, list_test_cases, get_metric, get_persona, list_reports, get_report, list_run_templates, list_scheduled_runs, get_scheduled_run",
+      "expected_output": "Retrieves Directory Review Baseline with exactly two cases: safe, scoped mild-headache guidance with escalation signs; and concise routine-appointment preparation without inventing a booking. Identifies Directory Review: Helpful and Safe as checking helpful, safe, appropriately scoped replies, and Standard Customer as a cooperative, on-topic en-US persona that waits for the agent. Lists report, template, and schedule summaries, and returns no more than 20 report rows and 20 schedule-history runs with explicit continuation or upstream-history-limit signals. Performs no writes and omits credentials and unrelated internal fields.",
       "expected_output_url": null
     },
     {
-      "description": "Create and refine independent disposable evaluation content.",
-      "user_prompt": "Generate one fresh UUID v4 for this attempt. Create a SCENARIO test set named OpenAI Reviewer Billing <that UUID>. Add exactly one test case where a customer disputes a duplicate charge, with expected behaviors to verify the duplicate and explain the next step. Then update only that test case's description to 'OpenAI reviewer disposable fixture.'",
+      "description": "Create and refine independent disposable evaluation content and a private report.",
+      "user_prompt": "Generate one fresh UUID v4 for this attempt. Create a SCENARIO test set named OpenAI Reviewer Billing <that UUID>. Add exactly one test case where a customer disputes a duplicate charge, with expected behaviors to verify the duplicate and explain the next step. Then update only that test case's description to 'OpenAI reviewer disposable fixture.' Separately create one uniquely named private saved report over completed run <PORTAL_COMPLETED_RUN_ID>. Do not request public visibility or modify any existing report.",
       "file_attachment_urls": null,
-      "tools_triggered": "create_test_set, create_test_case, update_test_case",
-      "expected_output": "After the required confirmations, creates exactly one test set named with a fresh collision-resistant UUID and one linked test case, then applies the requested description. Reports the resulting resource names without depending on another test case or changing anything else.",
+      "tools_triggered": "create_test_set, create_test_case, update_test_case, create_report",
+      "expected_output": "After the required confirmations, creates exactly one test set named with a fresh collision-resistant UUID and one linked test case, then applies the requested description. Also creates exactly one organization-private report over the specified completed run. Reports the resulting resource names without depending on another test case or changing anything else.",
       "expected_output_url": null
     },
     {
-      "description": "Inspect a stable reviewer agent and completed run without launching work.",
-      "user_prompt": "Retrieve the agent with reviewer fixture ID <PORTAL_AGENT_ID> and the completed run with reviewer fixture ID <PORTAL_COMPLETED_RUN_ID>. Summarize the agent's display name, model type, and endpoint-configured status, then report the run's status, progress, tags, and result overview. Do not create or update anything and do not start a run.",
+      "description": "Inspect a stable reviewer agent and completed run, then prepare a disabled schedule.",
+      "user_prompt": "Retrieve the agent with reviewer fixture ID <PORTAL_AGENT_ID> and the completed run with reviewer fixture ID <PORTAL_COMPLETED_RUN_ID>. Summarize the agent's display name, model type, and endpoint-configured status, then report the run's status, progress, tags, and result overview. Next, create one uniquely named weekday schedule from disposable template <PORTAL_RUN_TEMPLATE_ID> in timezone America/Los_Angeles with enabled false. Update only the new schedule's display name and keep it disabled.",
       "file_attachment_urls": null,
-      "tools_triggered": "get_agent, get_run",
-      "expected_output": "Reports Review Voice Agent as MODEL_TYPE_VOICE with endpoint_configured false. Reports the completed reviewer run as COMPLETED with 2/2 test cases completed, 0 failed, tags directory-review and openai-review, and Directory Review: Helpful and Safe = YES for both outputs. Performs no writes, does not depend on another test case, and starts no run.",
+      "tools_triggered": "get_agent, get_run, create_scheduled_run, update_scheduled_run",
+      "expected_output": "Reports Review Voice Agent as MODEL_TYPE_VOICE with endpoint_configured false. Reports the completed reviewer run as COMPLETED with 2/2 test cases completed, 0 failed, tags directory-review and openai-review, and Directory Review: Helpful and Safe = YES for both outputs. After the required confirmations, creates exactly one disabled schedule and changes only its display name without enabling it or triggering an evaluation.",
       "expected_output_url": null
     },
     {

docs/directory-submission.md

@@ -118,48 +118,42 @@ resources after review.
      required confirmations; no evaluation starts and the SIP address is never contacted.
    - Fixture: permission to create disposable agents. Generate a fresh UUID v4 or equivalent
      collision-resistant nonce and matching `sip:<nonce>@invalid.example` address for every attempt.
-2. **Stable evaluation setup inspection**
+2. **Stable evaluation, report, and schedule inspection**
    - Retrieve the portal-provided baseline test-set ID, only its test cases, the reviewer metric ID,
-     and the canonical reviewer persona ID.
-   - Expected behavior: retrieve only those exact resources. Make no changes and do not fall back
-     to mutable display-name discovery.
-   - Fixture: one baseline test set with two cases, one metric, and one canonical persona.
-3. **Independent disposable test content**
+     and the canonical reviewer persona ID. List reports, retrieve the stable report with a page
+     size of 20 and the reviewer metric filter, then list run templates and schedules and retrieve
+     the stable schedule with 20 recent runs.
+   - Expected behavior: retrieve only those exact resources. Report explicit continuation signals
+     for report rows and schedule history; make no changes and do not fall back to mutable
+     display-name discovery.
+   - Fixture: one baseline test set with two cases, one metric, one canonical persona, one stable
+     saved report, and one stable schedule with history.
+3. **Independent disposable test content and private report**
    - Create one SCENARIO test set named with a fresh UUID v4 or equivalent collision-resistant
      nonce, add one duplicate-charge test case, and
-     update only that case's description.
-   - Expected behavior: `create_test_set`, `create_test_case`, and `update_test_case` each run once
-     after confirmation. This case must not be reused by another submitted test.
-   - Fixture: permission to create disposable test data.
-4. **Stable agent and completed-run inspection**
-   - Retrieve the portal-provided stable agent ID and completed reviewer-run ID.
+     update only that case's description. Separately create one uniquely named, organization-private
+     report over the completed reviewer run.
+   - Expected behavior: `create_test_set`, `create_test_case`, `update_test_case`, and
+     `create_report` each run once after confirmation. This case must not be reused by another
+     submitted test; the new report must not accept public-sharing input.
+   - Fixture: permission to create disposable test data and one stable completed run.
+4. **Stable agent and completed-run inspection plus disabled schedule**
+   - Retrieve the portal-provided stable agent ID and completed reviewer-run ID. Create one
+     uniquely named weekday schedule from the disposable template with a concrete timezone and
+     without activation, then update only its display name while it remains disabled.
    - Expected behavior: call `get_agent` and `get_run`, report only the requested configuration and
-     result fields, and perform no writes or evaluation launch.
-   - Fixture: an independently valid agent and completed run that do not depend on cases 1 or 3.
+     result fields, then create and update exactly one disabled schedule without triggering an
+     evaluation.
+   - Fixture: an independently valid agent and completed run that do not depend on cases 1 or 3,
+     plus a disposable run template that can be used for a disabled schedule. Remove the disposable
+     schedule through the Coval app or API after review.
 5. **Bounded Sofia guidance**
    - Ask Sofia one standalone question containing only two Turn Count scores and a request for one
      task-completion metric.
    - Expected behavior: call `consult_sofia` once with only `prompt`; do not send prior turns or a
      caller session identifier.
    - Expected result: one task-success or task-completion metric recommendation and a concise
      rationale, with no writes.
-6. **Private saved report workflow**
-   - List saved reports, retrieve the portal-provided stable report with a page size of 20 and the
-     reviewer metric filter, then create one uniquely named report over the completed reviewer run.
-   - Expected behavior: the stable read returns no more than 20 rows and accurately indicates
-     whether more rows exist. The new report is organization-private and accepts no public-sharing
-     input.
-   - Fixture: one stable saved report, one stable completed run, and one reviewer metric.
-7. **Disabled recurring evaluation workflow**
-   - List run templates and schedules, retrieve the portal-provided schedule with 20 recent runs,
-     then create a uniquely named weekday schedule from the disposable template with a concrete
-     timezone and without activation. Update only its display name while it remains disabled.
-   - Expected behavior: history returns no more than 20 runs with a continuation token, or clearly
-     marks completeness unknown if the 500-run API window is exhausted. Creation sends
-     `enabled: false`; the update does not activate or trigger an evaluation.
-   - Fixture: one stable schedule with history, plus a disposable run template that can be used for
-     a disabled schedule. Remove the disposable schedule through the Coval app or API after review.
-
 ### Negative cases
 
 1. **Unrelated calendar request**
@@ -187,7 +181,7 @@ resources after review.
   annotations.
 - Replace any stale portal description with the client-neutral canonical copy in this repository;
   do not claim write support for metrics or personas.
-- Run all seven positive and three negative cases in fresh conversations with the clean reviewer
+- Run all five positive and three negative cases in fresh conversations with the clean reviewer
   account, record the exact tool sequence and result, and resolve every mismatch before submitting.
 - Use `https://app.coval.dev` as the allowed application link origin if link opening is enabled.
 - Do not add challenge tokens, credentials, or fixture IDs to this repository.

tests/unit/chatgpt-app-submission.test.ts

@@ -0,0 +1,91 @@
+import fs from 'node:fs';
+
+const submission = JSON.parse(
+  fs.readFileSync('chatgpt-app-submission.json', 'utf8'),
+) as {
+  tools: Record<
+    string,
+    {
+      annotations: {
+        readOnlyHint: boolean;
+        openWorldHint: boolean;
+        destructiveHint: boolean;
+      };
+    }
+  >;
+  test_cases: Array<{ tools_triggered: string }>;
+};
+
+describe('ChatGPT app submission artifact', () => {
+  it('keeps the positive review workflows within the portal limit', () => {
+    expect(submission.test_cases).toHaveLength(5);
+  });
+
+  it('declares the full production MCP tool catalog', () => {
+    expect(Object.keys(submission.tools).sort()).toEqual([
+      'consult_sofia',
+      'create_agent',
+      'create_report',
+      'create_run',
+      'create_scheduled_run',
+      'create_test_case',
+      'create_test_set',
+      'get_agent',
+      'get_metric',
+      'get_persona',
+      'get_report',
+      'get_run',
+      'get_scheduled_run',
+      'get_test_case',
+      'get_test_set',
+      'list_agents',
+      'list_metrics',
+      'list_personas',
+      'list_reports',
+      'list_run_templates',
+      'list_runs',
+      'list_scheduled_runs',
+      'list_test_cases',
+      'list_test_sets',
+      'update_agent',
+      'update_scheduled_run',
+      'update_test_case',
+    ]);
+  });
+
+  it('preserves the risk annotations for report and scheduling writes', () => {
+    expect(submission.tools.create_report.annotations).toEqual({
+      readOnlyHint: false,
+      openWorldHint: false,
+      destructiveHint: false,
+    });
+    for (const toolName of ['create_scheduled_run', 'update_scheduled_run']) {
+      expect(submission.tools[toolName].annotations).toEqual({
+        readOnlyHint: false,
+        openWorldHint: true,
+        destructiveHint: true,
+      });
+    }
+  });
+
+  it('includes independent report and scheduling reviewer workflows', () => {
+    const reviewTools = new Set(
+      submission.test_cases.flatMap(({ tools_triggered }) =>
+        tools_triggered.split(', ').filter(Boolean),
+      ),
+    );
+
+    for (const toolName of [
+      'list_reports',
+      'get_report',
+      'create_report',
+      'list_run_templates',
+      'list_scheduled_runs',
+      'get_scheduled_run',
+      'create_scheduled_run',
+      'update_scheduled_run',
+    ]) {
+      expect(reviewTools.has(toolName)).toBe(true);
+    }
+  });
+});