diff --git a/chatgpt-app-submission.json b/chatgpt-app-submission.template.json similarity index 71% rename from chatgpt-app-submission.json rename to chatgpt-app-submission.template.json index b92adc1..0727f36 100644 --- a/chatgpt-app-submission.json +++ b/chatgpt-app-submission.template.json @@ -1,5 +1,5 @@ { - "$schema": "https://developers.openai.com/apps-sdk/schemas/chatgpt-app-submission.v1.json", + "$schema": "https://developers.openai.com/plugins/schemas/chatgpt-app-submission.v1.json", "schema_version": 1, "app_info": { "display_name": "Coval", @@ -235,6 +235,102 @@ "open_world_justification": "Does not publish content or change public or third-party state.", "destructive_justification": "Cannot create, modify, run, or delete resources." } + }, + "list_reports": { + "annotations": { + "readOnlyHint": true, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Only retrieves saved report summaries from the authenticated Coval organization without changing them.", + "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.", + "destructive_justification": "Does not delete, overwrite, or change any report." + } + }, + "get_report": { + "annotations": { + "readOnlyHint": true, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Only retrieves one saved report and a bounded page of its results from the authenticated Coval organization.", + "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.", + "destructive_justification": "Does not delete, overwrite, or change the selected report." + } + }, + "create_report": { + "annotations": { + "readOnlyHint": false, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Creates an organization-private saved report over explicitly identified evaluation runs.", + "open_world_justification": "Creates only a private report in the authenticated Coval organization and cannot publish it publicly.", + "destructive_justification": "Creates a new private report without deleting or overwriting an existing resource." + } + }, + "list_run_templates": { + "annotations": { + "readOnlyHint": true, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Only retrieves reusable evaluation configurations from the authenticated Coval organization.", + "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.", + "destructive_justification": "Does not delete, overwrite, or change any run template." + } + }, + "list_scheduled_runs": { + "annotations": { + "readOnlyHint": true, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Only retrieves recurring evaluation schedule summaries from the authenticated Coval organization.", + "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.", + "destructive_justification": "Does not delete, overwrite, or change any schedule." + } + }, + "get_scheduled_run": { + "annotations": { + "readOnlyHint": true, + "openWorldHint": false, + "destructiveHint": false + }, + "justifications": { + "read_only_justification": "Only retrieves one recurring evaluation schedule and a bounded page of its history.", + "open_world_justification": "Does not publish content or change state outside the authenticated Coval organization.", + "destructive_justification": "Does not delete, overwrite, or change the selected schedule." + } + }, + "create_scheduled_run": { + "annotations": { + "readOnlyHint": false, + "openWorldHint": true, + "destructiveHint": true + }, + "justifications": { + "read_only_justification": "Creates a recurring evaluation schedule from a selected run template in the authenticated Coval organization.", + "open_world_justification": "A schedule can later trigger evaluations that contact configured agents, phone numbers, webhooks, or other external endpoints.", + "destructive_justification": "Creates recurring configuration that can trigger future evaluation activity and costs." + } + }, + "update_scheduled_run": { + "annotations": { + "readOnlyHint": false, + "openWorldHint": true, + "destructiveHint": true + }, + "justifications": { + "read_only_justification": "Updates selected fields on one recurring evaluation schedule in the authenticated Coval organization.", + "open_world_justification": "Enabling or changing the timing of a schedule can trigger evaluations that contact configured external endpoints.", + "destructive_justification": "Overwrites selected schedule fields and can change future evaluation activity and costs." + } } }, "test_cases": [ @@ -247,27 +343,27 @@ "expected_output_url": null }, { - "description": "Inspect the stable reviewer evaluation setup without changing it.", - "user_prompt": "Retrieve the test set with reviewer fixture ID and list only the test cases in that test set. Also retrieve the metric with reviewer fixture ID and the persona with reviewer fixture ID . Summarize the scenarios, expected behaviors, metric purpose, and persona behavior without changing anything.", + "description": "Inspect stable reviewer evaluation, report, and schedule resources without changing them.", + "user_prompt": "Retrieve the test set with reviewer fixture ID and list only the test cases in that test set. Also retrieve the metric with reviewer fixture ID and the persona with reviewer fixture ID . List saved reports and retrieve report with page_size 20 and metric ID . List run templates and recurring schedules, then retrieve schedule with history_size 20. Summarize the resources and all continuation signals without changing anything.", "file_attachment_urls": null, - "tools_triggered": "get_test_set, list_test_cases, get_metric, get_persona", - "expected_output": "Retrieves Directory Review Baseline with exactly two cases: safe, scoped mild-headache guidance with escalation signs; and concise routine-appointment preparation without inventing a booking. Identifies Directory Review: Helpful and Safe as checking helpful, safe, appropriately scoped replies, and Standard Customer as a cooperative, on-topic en-US persona that waits for the agent. Performs no writes and omits credentials and unrelated internal fields.", + "tools_triggered": "get_test_set, list_test_cases, get_metric, get_persona, list_reports, get_report, list_run_templates, list_scheduled_runs, get_scheduled_run", + "expected_output": "Retrieves Directory Review Baseline with exactly two cases: safe, scoped mild-headache guidance with escalation signs; and concise routine-appointment preparation without inventing a booking. Identifies Directory Review: Helpful and Safe as checking helpful, safe, appropriately scoped replies, and Standard Customer as a cooperative, on-topic en-US persona that waits for the agent. Lists report, template, and schedule summaries, and returns no more than 20 report rows and 20 schedule-history runs with explicit continuation or upstream-history-limit signals. Performs no writes and omits credentials and unrelated internal fields.", "expected_output_url": null }, { - "description": "Create and refine independent disposable evaluation content.", - "user_prompt": "Generate one fresh UUID v4 for this attempt. Create a SCENARIO test set named OpenAI Reviewer Billing . Add exactly one test case where a customer disputes a duplicate charge, with expected behaviors to verify the duplicate and explain the next step. Then update only that test case's description to 'OpenAI reviewer disposable fixture.'", + "description": "Create and refine independent disposable evaluation content and a private report.", + "user_prompt": "Generate one fresh UUID v4 for this attempt. Create a SCENARIO test set named OpenAI Reviewer Billing . Add exactly one test case where a customer disputes a duplicate charge, with expected behaviors to verify the duplicate and explain the next step. Then update only that test case's description to 'OpenAI reviewer disposable fixture.' Separately create one uniquely named private saved report over completed run . Do not request public visibility or modify any existing report.", "file_attachment_urls": null, - "tools_triggered": "create_test_set, create_test_case, update_test_case", - "expected_output": "After the required confirmations, creates exactly one test set named with a fresh collision-resistant UUID and one linked test case, then applies the requested description. Reports the resulting resource names without depending on another test case or changing anything else.", + "tools_triggered": "create_test_set, create_test_case, update_test_case, create_report", + "expected_output": "After the required confirmations, creates exactly one test set named with a fresh collision-resistant UUID and one linked test case, then applies the requested description. Also creates exactly one organization-private report over the specified completed run. Reports the resulting resource names without depending on another test case or changing anything else.", "expected_output_url": null }, { - "description": "Inspect a stable reviewer agent and completed run without launching work.", - "user_prompt": "Retrieve the agent with reviewer fixture ID and the completed run with reviewer fixture ID . Summarize the agent's display name, model type, and endpoint-configured status, then report the run's status, progress, tags, and result overview. Do not create or update anything and do not start a run.", + "description": "Inspect a stable reviewer agent and completed run, then prepare a disabled schedule.", + "user_prompt": "Retrieve the agent with reviewer fixture ID and the completed run with reviewer fixture ID . Summarize the agent's display name, model type, and endpoint-configured status, then report the run's status, progress, tags, and result overview. Next, create one uniquely named weekday schedule from disposable template in timezone America/Los_Angeles with enabled false. Update only the new schedule's display name and keep it disabled.", "file_attachment_urls": null, - "tools_triggered": "get_agent, get_run", - "expected_output": "Reports Review Voice Agent as MODEL_TYPE_VOICE with endpoint_configured false. Reports the completed reviewer run as COMPLETED with 2/2 test cases completed, 0 failed, tags directory-review and openai-review, and Directory Review: Helpful and Safe = YES for both outputs. Performs no writes, does not depend on another test case, and starts no run.", + "tools_triggered": "get_agent, get_run, create_scheduled_run, update_scheduled_run", + "expected_output": "Reports Review Voice Agent as MODEL_TYPE_VOICE with endpoint_configured false. Reports the completed reviewer run as COMPLETED with 2/2 test cases completed, 0 failed, tags directory-review and openai-review, and Directory Review: Helpful and Safe = YES for both outputs. After the required confirmations, creates exactly one disabled schedule and changes only its display name without enabling it or triggering an evaluation.", "expected_output_url": null }, { diff --git a/docs/directory-submission.md b/docs/directory-submission.md index 2579bf3..e7d0817 100644 --- a/docs/directory-submission.md +++ b/docs/directory-submission.md @@ -100,14 +100,28 @@ write or destructive annotations as applicable. ## OpenAI review test cases -The review account should use a populated, disposable Coval organization. Provide concrete fixture -IDs and credentials only through the submission portal. Replace every `PORTAL_*` placeholder in the -submission artifact with the corresponding exact fixture ID before submitting. Replace generic -expected results for stable fixtures with their exact names, configuration, completion state, -progress, tags, and result summary so the reviewer can verify them objectively. Keep every case -independently runnable: no case may depend on a resource created by another case, a moving "most -recent" target, or a fixed-name disposable resource left by an earlier run. Reset disposable -resources after review. +The review account should use a populated, disposable Coval organization. The checked-in +`chatgpt-app-submission.template.json` is intentionally a template: it contains `PORTAL_*` fixture +placeholders and must never be uploaded or contain real reviewer IDs or credentials. Materialize an +untracked export outside this repository, then run its preflight before uploading it: + +```bash +npm run submission:materialize -- \ + --fixtures /secure/path/reviewer-fixtures.json \ + --output /secure/path/chatgpt-app-submission.review.json +npm run submission:preflight -- /secure/path/chatgpt-app-submission.review.json +``` + +The fixture file maps each required `PORTAL_*` token to a concrete reviewer fixture ID. Keep it +outside the repository. The materializer rejects missing, extra, or malformed fixture values; the +preflight rejects any unresolved placeholders, validates the current published OpenAI schema, and +compares the submitted tool catalog and annotations with the locally built OpenAI MCP profile. + +Replace generic expected results for stable fixtures with their exact names, configuration, +completion state, progress, tags, and result summary so the reviewer can verify them objectively. +Keep every case independently runnable: no case may depend on a resource created by another case, a +moving "most recent" target, or a fixed-name disposable resource left by an earlier run. Reset +disposable resources after review. ### Positive cases @@ -118,24 +132,35 @@ resources after review. required confirmations; no evaluation starts and the SIP address is never contacted. - Fixture: permission to create disposable agents. Generate a fresh UUID v4 or equivalent collision-resistant nonce and matching `sip:@invalid.example` address for every attempt. -2. **Stable evaluation setup inspection** +2. **Stable evaluation, report, and schedule inspection** - Retrieve the portal-provided baseline test-set ID, only its test cases, the reviewer metric ID, - and the canonical reviewer persona ID. - - Expected behavior: retrieve only those exact resources. Make no changes and do not fall back - to mutable display-name discovery. - - Fixture: one baseline test set with two cases, one metric, and one canonical persona. -3. **Independent disposable test content** + and the canonical reviewer persona ID. List reports, retrieve the stable report with a page + size of 20 and the reviewer metric filter, then list run templates and schedules and retrieve + the stable schedule with 20 recent runs. + - Expected behavior: retrieve only those exact resources. Report explicit continuation signals + for report rows and schedule history; make no changes and do not fall back to mutable + display-name discovery. + - Fixture: one baseline test set with two cases, one metric, one canonical persona, one stable + saved report, and one stable schedule with history. +3. **Independent disposable test content and private report** - Create one SCENARIO test set named with a fresh UUID v4 or equivalent collision-resistant nonce, add one duplicate-charge test case, and - update only that case's description. - - Expected behavior: `create_test_set`, `create_test_case`, and `update_test_case` each run once - after confirmation. This case must not be reused by another submitted test. - - Fixture: permission to create disposable test data. -4. **Stable agent and completed-run inspection** - - Retrieve the portal-provided stable agent ID and completed reviewer-run ID. + update only that case's description. Separately create one uniquely named, organization-private + report over the completed reviewer run. + - Expected behavior: `create_test_set`, `create_test_case`, `update_test_case`, and + `create_report` each run once after confirmation. This case must not be reused by another + submitted test; the new report must not accept public-sharing input. + - Fixture: permission to create disposable test data and one stable completed run. +4. **Stable agent and completed-run inspection plus disabled schedule** + - Retrieve the portal-provided stable agent ID and completed reviewer-run ID. Create one + uniquely named weekday schedule from the disposable template with a concrete timezone and + without activation, then update only its display name while it remains disabled. - Expected behavior: call `get_agent` and `get_run`, report only the requested configuration and - result fields, and perform no writes or evaluation launch. - - Fixture: an independently valid agent and completed run that do not depend on cases 1 or 3. + result fields, then create and update exactly one disabled schedule without triggering an + evaluation. + - Fixture: an independently valid agent and completed run that do not depend on cases 1 or 3, + plus a disposable run template that can be used for a disabled schedule. Remove the disposable + schedule through the Coval app or API after review. 5. **Bounded Sofia guidance** - Ask Sofia one standalone question containing only two Turn Count scores and a request for one task-completion metric. @@ -143,23 +168,6 @@ resources after review. caller session identifier. - Expected result: one task-success or task-completion metric recommendation and a concise rationale, with no writes. -6. **Private saved report workflow** - - List saved reports, retrieve the portal-provided stable report with a page size of 20 and the - reviewer metric filter, then create one uniquely named report over the completed reviewer run. - - Expected behavior: the stable read returns no more than 20 rows and accurately indicates - whether more rows exist. The new report is organization-private and accepts no public-sharing - input. - - Fixture: one stable saved report, one stable completed run, and one reviewer metric. -7. **Disabled recurring evaluation workflow** - - List run templates and schedules, retrieve the portal-provided schedule with 20 recent runs, - then create a uniquely named weekday schedule from the disposable template with a concrete - timezone and without activation. Update only its display name while it remains disabled. - - Expected behavior: history returns no more than 20 runs with a continuation token, or clearly - marks completeness unknown if the 500-run API window is exhausted. Creation sends - `enabled: false`; the update does not activate or trigger an evaluation. - - Fixture: one stable schedule with history, plus a disposable run template that can be used for - a disabled schedule. Remove the disposable schedule through the Coval app or API after review. - ### Negative cases 1. **Unrelated calendar request** @@ -179,7 +187,8 @@ resources after review. - Set `OPENAI_APPS_CHALLENGE` to the exact token generated by the OpenAI submission portal, deploy, and verify the well-known endpoint returns only that token before selecting **Verify**. -- Supply reviewer credentials and fixture IDs through the platform's secure submission fields. +- Supply reviewer credentials through the platform's secure submission fields. Materialize fixture + IDs in the untracked submission export; do not upload the checked-in template. - Select the verified Coval developer or business identity; do not submit while the portal shows `No Identity Selected`. - After deploying the exact reviewed server head, run **Scan Tools** again on the submitted app @@ -187,8 +196,9 @@ resources after review. annotations. - Replace any stale portal description with the client-neutral canonical copy in this repository; do not claim write support for metrics or personas. -- Run all seven positive and three negative cases in fresh conversations with the clean reviewer - account, record the exact tool sequence and result, and resolve every mismatch before submitting. +- Run all five positive and three negative cases in fresh conversations with the clean reviewer + account on ChatGPT web and mobile. Record the exact tool sequence and result, reset disposable + resources, and repeat the fixture-dependent positive cases once before submitting. - Use `https://app.coval.dev` as the allowed application link origin if link opening is enabled. - Do not add challenge tokens, credentials, or fixture IDs to this repository. diff --git a/package-lock.json b/package-lock.json index e938305..21b4331 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "@types/node": "^22.14.0", "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", + "ajv": "^8.20.0", "eslint": "^8.57.0", "jest": "^29.7.0", "ts-jest": "^29.3.1", @@ -1173,6 +1174,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -1184,6 +1202,13 @@ "concat-map": "0.0.1" } }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/eslintrc/node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1861,6 +1886,30 @@ "mcp-inspector-client": "bin/start.js" } }, + "node_modules/@modelcontextprotocol/inspector-client/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@modelcontextprotocol/inspector-client/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@modelcontextprotocol/inspector-server": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/inspector-server/-/inspector-server-0.14.3.tgz", @@ -1918,28 +1967,6 @@ } } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/@modelcontextprotocol/sdk/node_modules/pkce-challenge": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", @@ -3498,16 +3525,15 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -3531,28 +3557,6 @@ } } }, - "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -4753,6 +4757,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -4764,6 +4785,13 @@ "concat-map": "0.0.1" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/eslint/node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -6544,10 +6572,9 @@ "license": "MIT" }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, "node_modules/json-schema-typed": { diff --git a/package.json b/package.json index 9f740f6..8192c9f 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "inspector": "npx @modelcontextprotocol/inspector", "check:remote": "node scripts/check-remote.mjs", "qualify:image": "node scripts/qualify-image.mjs", + "submission:materialize": "node scripts/materialize-chatgpt-submission.mjs", + "submission:preflight": "npm run build && node scripts/preflight-chatgpt-submission.mjs", "test": "NODE_OPTIONS=--experimental-vm-modules jest", "test:unit": "npm test -- --testPathPattern=unit", "test:e2e": "npm test -- --testPathPattern=e2e", @@ -66,6 +68,7 @@ "@types/node": "^22.14.0", "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", + "ajv": "^8.20.0", "eslint": "^8.57.0", "jest": "^29.7.0", "ts-jest": "^29.3.1", diff --git a/scripts/materialize-chatgpt-submission.mjs b/scripts/materialize-chatgpt-submission.mjs new file mode 100644 index 0000000..15995fe --- /dev/null +++ b/scripts/materialize-chatgpt-submission.mjs @@ -0,0 +1,146 @@ +import { chmod, lstat, readFile, realpath, writeFile } from 'node:fs/promises'; +import { basename, dirname, relative, resolve, sep } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const repositoryRoot = resolve(fileURLToPath(new URL('..', import.meta.url))); +const templatePath = resolve(repositoryRoot, 'chatgpt-app-submission.template.json'); +const fixtureTokenPattern = /^PORTAL_[A-Z_]+$/; +const fixtureValuePattern = /^[A-Za-z0-9_-]+$/; + +function usage() { + throw new Error( + 'Usage: npm run submission:materialize -- --fixtures --output ', + ); +} + +export function parseMaterializeArguments(arguments_) { + if (arguments_.length !== 4) usage(); + const values = new Map(); + for (let index = 0; index < arguments_.length; index += 2) { + const option = arguments_[index]; + const value = arguments_[index + 1]; + if (!['--fixtures', '--output'].includes(option) || !value || values.has(option)) usage(); + values.set(option, value); + } + return { + fixturesPath: resolve(values.get('--fixtures')), + outputPath: resolve(values.get('--output')), + }; +} + +export function isOutsideRepository(path, canonicalRepositoryRoot) { + const pathFromRepository = relative(canonicalRepositoryRoot, path); + return pathFromRepository === '..' || pathFromRepository.startsWith(`..${sep}`); +} + +async function readJson(path, label, readFileImpl) { + try { + return JSON.parse(await readFileImpl(path, 'utf8')); + } catch (error) { + throw new Error( + `Unable to read ${label}: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +export function fixtureTokens(template) { + return [...new Set(JSON.stringify(template).match(//g) ?? [])] + .map((placeholder) => placeholder.slice(1, -1)) + .sort(); +} + +export function validateFixtures(fixtures, expectedTokens) { + if (!fixtures || typeof fixtures !== 'object' || Array.isArray(fixtures)) { + throw new Error('Fixture JSON must map PORTAL_* tokens to reviewer fixture IDs.'); + } + + const keys = Object.keys(fixtures).sort(); + const invalidKeys = keys.filter((key) => !fixtureTokenPattern.test(key)); + const missing = expectedTokens.filter((key) => !keys.includes(key)); + const extras = keys.filter((key) => !expectedTokens.includes(key)); + if (invalidKeys.length || missing.length || extras.length) { + throw new Error( + `Fixture keys must exactly match the template tokens. Missing: ${missing.join(', ') || 'none'}; ` + + `extra: ${extras.join(', ') || 'none'}; invalid: ${invalidKeys.join(', ') || 'none'}.`, + ); + } + + for (const [key, value] of Object.entries(fixtures)) { + if (typeof value !== 'string' || !fixtureValuePattern.test(value)) { + throw new Error( + `${key} must be a non-empty Coval resource ID containing only letters, digits, hyphens, or underscores.`, + ); + } + } +} + +async function canonicalOutputPath(outputPath, lstatImpl, realpathImpl) { + try { + const outputStatus = await lstatImpl(outputPath); + if (outputStatus.isSymbolicLink()) { + throw new Error('Materialized submission output must not be a symlink.'); + } + } catch (error) { + if (!(error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')) { + throw error; + } + } + + const canonicalParent = await realpathImpl(dirname(outputPath)); + return resolve(canonicalParent, basename(outputPath)); +} + +export async function materializeSubmission({ + fixturesPath, + outputPath, + repositoryPath = repositoryRoot, + template = templatePath, + dependencies = { chmod, lstat, readFile, realpath, writeFile }, +}) { + const canonicalRepositoryRoot = await dependencies.realpath(repositoryPath); + const canonicalFixturesPath = await dependencies.realpath(fixturesPath); + if (!isOutsideRepository(canonicalFixturesPath, canonicalRepositoryRoot)) { + throw new Error('Keep reviewer fixture IDs outside the repository so they cannot be committed.'); + } + + const canonicalOutputPathValue = await canonicalOutputPath( + outputPath, + dependencies.lstat, + dependencies.realpath, + ); + if (!isOutsideRepository(canonicalOutputPathValue, canonicalRepositoryRoot)) { + throw new Error( + 'Write the materialized submission outside the repository so reviewer fixture IDs cannot be committed.', + ); + } + + const submissionTemplate = await readJson(template, 'submission template', dependencies.readFile); + const fixtures = await readJson(canonicalFixturesPath, 'fixture JSON', dependencies.readFile); + const tokens = fixtureTokens(submissionTemplate); + validateFixtures(fixtures, tokens); + + const materialized = JSON.parse( + JSON.stringify(submissionTemplate).replace( + //g, + (placeholder) => fixtures[placeholder.slice(1, -1)], + ), + ); + + await dependencies.writeFile(canonicalOutputPathValue, `${JSON.stringify(materialized, null, 2)}\n`, { + mode: 0o600, + }); + await dependencies.chmod(canonicalOutputPathValue, 0o600); + return { outputPath: canonicalOutputPathValue, tokenCount: tokens.length }; +} + +async function main() { + const { fixturesPath, outputPath } = parseMaterializeArguments(process.argv.slice(2)); + const result = await materializeSubmission({ fixturesPath, outputPath }); + console.log( + `Materialized ${result.tokenCount} reviewer fixture IDs to ${result.outputPath}. Run npm run submission:preflight -- ${result.outputPath} before upload.`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + await main(); +} diff --git a/scripts/preflight-chatgpt-submission.mjs b/scripts/preflight-chatgpt-submission.mjs new file mode 100644 index 0000000..7d93a01 --- /dev/null +++ b/scripts/preflight-chatgpt-submission.mjs @@ -0,0 +1,156 @@ +import Ajv2020 from 'ajv/dist/2020.js'; +import { readFile, realpath } from 'node:fs/promises'; +import { relative, resolve, sep } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const repositoryRoot = resolve(fileURLToPath(new URL('..', import.meta.url))); +export const publishedSchemaUrl = + 'https://developers.openai.com/plugins/schemas/chatgpt-app-submission.v1.json'; +const placeholderPattern = //; + +function usage() { + throw new Error( + 'Usage: npm run submission:preflight -- ', + ); +} + +export function parsePreflightArguments(arguments_) { + if (arguments_.length !== 1 || !arguments_[0]) usage(); + return resolve(arguments_[0]); +} + +export function isOutsideRepository(path, canonicalRepositoryRoot) { + const pathFromRepository = relative(canonicalRepositoryRoot, path); + return pathFromRepository === '..' || pathFromRepository.startsWith(`..${sep}`); +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +export function walkStrings(value, path = '$') { + if (typeof value === 'string') return [{ path, value }]; + if (Array.isArray(value)) { + return value.flatMap((entry, index) => walkStrings(entry, `${path}[${index}]`)); + } + if (!value || typeof value !== 'object') return []; + return Object.entries(value).flatMap(([key, entry]) => walkStrings(entry, `${path}.${key}`)); +} + +async function builtSourceToolCatalog() { + const { registerAllTools } = await import('../dist/tools/index.js'); + const registrations = new Map(); + registerAllTools( + { + registerTool(name, configuration) { + registrations.set(name, configuration); + }, + }, + {}, + { annotationProfile: 'standard', inputProfile: 'openai', includeSofia: true }, + ); + return registrations; +} + +async function publishedSchema(fetchImpl) { + const response = await fetchImpl(publishedSchemaUrl, { signal: AbortSignal.timeout(15_000) }); + if (!response.ok) throw new Error(`Published OpenAI schema returned ${response.status}.`); + return response.json(); +} + +export async function preflightSubmission({ + submissionPath, + repositoryPath = repositoryRoot, + dependencies = { readFile, realpath }, + fetchImpl = fetch, + sourceToolCatalog = builtSourceToolCatalog, +}) { + const canonicalRepositoryRoot = await dependencies.realpath(repositoryPath); + const canonicalSubmissionPath = await dependencies.realpath(submissionPath); + if (!isOutsideRepository(canonicalSubmissionPath, canonicalRepositoryRoot)) { + throw new Error( + 'Preflight only accepts a materialized submission outside the repository, never the checked-in template.', + ); + } + + let submission; + try { + submission = JSON.parse(await dependencies.readFile(canonicalSubmissionPath, 'utf8')); + } catch (error) { + throw new Error( + `Unable to read materialized submission: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + assert(submission.$schema === publishedSchemaUrl, `Submission must declare ${publishedSchemaUrl}.`); + const unresolved = walkStrings(submission).filter(({ value }) => placeholderPattern.test(value)); + assert( + unresolved.length === 0, + `Submission still contains reviewer fixture placeholders at ${unresolved.map(({ path }) => path).join(', ')}.`, + ); + + const ajv = new Ajv2020({ allErrors: true, strict: false }); + const schema = await publishedSchema(fetchImpl); + const valid = ajv.validate(schema, submission); + assert( + valid, + `Submission does not match the current OpenAI schema: ${ajv.errorsText(ajv.errors, { separator: '; ' })}`, + ); + + assert( + Array.isArray(submission.test_cases) && submission.test_cases.length >= 5, + 'Submission must include at least five positive test cases.', + ); + assert( + Array.isArray(submission.negative_test_cases) && submission.negative_test_cases.length >= 3, + 'Submission must include at least three negative test cases.', + ); + for (const [index, testCase] of submission.test_cases.entries()) { + assert( + typeof testCase.expected_output === 'string' && testCase.expected_output.trim(), + `Positive case ${index + 1} needs a concrete expected_output.`, + ); + } + for (const [index, testCase] of submission.negative_test_cases.entries()) { + assert( + typeof testCase.expected_output === 'string' && testCase.expected_output.trim(), + `Negative case ${index + 1} needs a concrete expected_output.`, + ); + } + + const sourceTools = await sourceToolCatalog(); + const submissionToolNames = Object.keys(submission.tools ?? {}).sort(); + const sourceToolNames = [...sourceTools.keys()].sort(); + assert( + JSON.stringify(submissionToolNames) === JSON.stringify(sourceToolNames), + `Submission tool catalog differs from the OpenAI MCP profile. Submission: ${submissionToolNames.join(', ')}. Source: ${sourceToolNames.join(', ')}.`, + ); + for (const toolName of sourceToolNames) { + const submittedAnnotations = submission.tools[toolName]?.annotations; + const sourceAnnotations = sourceTools.get(toolName)?.annotations; + for (const annotation of ['readOnlyHint', 'openWorldHint', 'destructiveHint']) { + assert( + submittedAnnotations?.[annotation] === sourceAnnotations?.[annotation], + `${toolName} ${annotation} does not match the OpenAI MCP profile.`, + ); + } + } + + return { + negativeCaseCount: submission.negative_test_cases.length, + positiveCaseCount: submission.test_cases.length, + toolCount: submissionToolNames.length, + }; +} + +async function main() { + const submissionPath = parsePreflightArguments(process.argv.slice(2)); + const result = await preflightSubmission({ submissionPath }); + console.log( + `Submission preflight passed: ${result.toolCount} tools, ${result.positiveCaseCount} positive cases, ${result.negativeCaseCount} negative cases, and no unresolved fixture placeholders.`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + await main(); +} diff --git a/tests/unit/chatgpt-app-submission.test.ts b/tests/unit/chatgpt-app-submission.test.ts new file mode 100644 index 0000000..86dca7c --- /dev/null +++ b/tests/unit/chatgpt-app-submission.test.ts @@ -0,0 +1,123 @@ +import fs from 'node:fs'; + +const submission = JSON.parse( + fs.readFileSync('chatgpt-app-submission.template.json', 'utf8'), +) as { + $schema: string; + tools: Record< + string, + { + annotations: { + readOnlyHint: boolean; + openWorldHint: boolean; + destructiveHint: boolean; + }; + } + >; + test_cases: Array<{ + expected_output: string | null; + tools_triggered: string; + user_prompt: string; + }>; +}; + +describe('ChatGPT app submission artifact', () => { + it('keeps reviewer fixture IDs only in the materialized portal export', () => { + expect(submission.$schema).toBe( + 'https://developers.openai.com/plugins/schemas/chatgpt-app-submission.v1.json', + ); + + const placeholders = submission.test_cases.flatMap(({ user_prompt }) => + [...user_prompt.matchAll(//g)].map(([placeholder]) => placeholder), + ); + + expect([...new Set(placeholders)].sort()).toEqual([ + '', + '', + '', + '', + '', + '', + '', + '', + ]); + }); + + it('keeps the positive review workflows within the portal limit', () => { + expect(submission.test_cases).toHaveLength(5); + }); + + it('documents a concrete expected result for every positive workflow', () => { + for (const testCase of submission.test_cases) { + expect(testCase.expected_output?.trim()).toBeTruthy(); + } + }); + + it('declares the full production MCP tool catalog', () => { + expect(Object.keys(submission.tools).sort()).toEqual([ + 'consult_sofia', + 'create_agent', + 'create_report', + 'create_run', + 'create_scheduled_run', + 'create_test_case', + 'create_test_set', + 'get_agent', + 'get_metric', + 'get_persona', + 'get_report', + 'get_run', + 'get_scheduled_run', + 'get_test_case', + 'get_test_set', + 'list_agents', + 'list_metrics', + 'list_personas', + 'list_reports', + 'list_run_templates', + 'list_runs', + 'list_scheduled_runs', + 'list_test_cases', + 'list_test_sets', + 'update_agent', + 'update_scheduled_run', + 'update_test_case', + ]); + }); + + it('preserves the risk annotations for report and scheduling writes', () => { + expect(submission.tools.create_report.annotations).toEqual({ + readOnlyHint: false, + openWorldHint: false, + destructiveHint: false, + }); + for (const toolName of ['create_scheduled_run', 'update_scheduled_run']) { + expect(submission.tools[toolName].annotations).toEqual({ + readOnlyHint: false, + openWorldHint: true, + destructiveHint: true, + }); + } + }); + + it('includes independent report and scheduling reviewer workflows', () => { + const reviewTools = new Set( + submission.test_cases.flatMap(({ tools_triggered }) => + tools_triggered.split(', ').filter(Boolean), + ), + ); + + for (const toolName of [ + 'list_reports', + 'get_report', + 'create_report', + 'list_run_templates', + 'list_scheduled_runs', + 'get_scheduled_run', + 'create_scheduled_run', + 'update_scheduled_run', + ]) { + expect(reviewTools.has(toolName)).toBe(true); + } + }); +}); diff --git a/tests/unit/chatgpt-submission-scripts.test.ts b/tests/unit/chatgpt-submission-scripts.test.ts new file mode 100644 index 0000000..65a29c6 --- /dev/null +++ b/tests/unit/chatgpt-submission-scripts.test.ts @@ -0,0 +1,197 @@ +import { + copyFile, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const templatePath = fileURLToPath( + new URL('../../chatgpt-app-submission.template.json', import.meta.url), +); +const template = JSON.parse(await readFile(templatePath, 'utf8')) as { + $schema: string; + negative_test_cases: unknown[]; + test_cases: Array<{ user_prompt: string }>; + tools: Record }>; +}; +const materializer = await import( + new URL('../../scripts/materialize-chatgpt-submission.mjs', import.meta.url).href, +); +const preflight = await import( + new URL('../../scripts/preflight-chatgpt-submission.mjs', import.meta.url).href, +); + +const schema = { $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'object' }; + +function matchingSourceToolCatalog() { + return async () => + new Map( + Object.entries(template.tools).map(([name, tool]) => [name, { annotations: tool.annotations }]), + ); +} + +function schemaFetch(schemaToReturn = schema) { + return async () => + new Response(JSON.stringify(schemaToReturn), { + headers: { 'content-type': 'application/schema+json' }, + status: 200, + }); +} + +function fixtures() { + const tokens = new Set( + template.test_cases.flatMap(({ user_prompt }) => + [...user_prompt.matchAll(//g)].map(([, token]) => `PORTAL_${token}`), + ), + ); + + return Object.fromEntries( + [...tokens].map((token) => [ + token, + token === 'PORTAL_TEST_SET_ID' ? 'ReviewSet' : `review_${token.slice('PORTAL_'.length).toLowerCase()}`, + ]), + ); +} + +async function writeJson(path: string, value: unknown) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +describe('ChatGPT submission scripts', () => { + let temporaryDirectory: string; + + beforeEach(async () => { + temporaryDirectory = await mkdtemp(join(tmpdir(), 'coval-chatgpt-submission-')); + }); + + afterEach(async () => { + await rm(temporaryDirectory, { force: true, recursive: true }); + }); + + it('rejects incomplete or malformed materializer arguments', () => { + expect(() => materializer.parseMaterializeArguments([])).toThrow('Usage:'); + expect(() => materializer.parseMaterializeArguments(['--fixtures', 'fixtures.json'])).toThrow( + 'Usage:', + ); + expect(() => preflight.parsePreflightArguments([])).toThrow('Usage:'); + }); + + it('materializes and preflights a complete outside-repository submission', async () => { + const fixturesPath = join(temporaryDirectory, 'fixtures.json'); + const outputPath = join(temporaryDirectory, 'submission.json'); + await writeJson(fixturesPath, fixtures()); + + const result = await materializer.materializeSubmission({ fixturesPath, outputPath }); + expect(result.tokenCount).toBe(8); + expect(JSON.stringify(JSON.parse(await readFile(outputPath, 'utf8')))).not.toContain(' { + const fixturesPath = join(temporaryDirectory, 'fixtures.json'); + await writeJson(fixturesPath, { ...fixtures(), PORTAL_AGENT_ID: 'bad/id' }); + + await expect( + materializer.materializeSubmission({ + fixturesPath, + outputPath: join(temporaryDirectory, 'submission.json'), + }), + ).rejects.toThrow('PORTAL_AGENT_ID must be a non-empty Coval resource ID'); + + await writeJson(fixturesPath, { PORTAL_AGENT_ID: 'review_agent' }); + await expect( + materializer.materializeSubmission({ + fixturesPath, + outputPath: join(temporaryDirectory, 'submission.json'), + }), + ).rejects.toThrow('Fixture keys must exactly match'); + }); + + it('rejects repository paths and symlinks that could store fixture data in the repository', async () => { + const externalFixturesPath = join(temporaryDirectory, 'fixtures.json'); + await writeJson(externalFixturesPath, fixtures()); + + await expect( + materializer.materializeSubmission({ + fixturesPath: templatePath, + outputPath: join(temporaryDirectory, 'submission.json'), + }), + ).rejects.toThrow('outside the repository'); + + const fixtureSymlink = join(temporaryDirectory, 'fixtures-link.json'); + await symlink(templatePath, fixtureSymlink); + await expect( + materializer.materializeSubmission({ + fixturesPath: fixtureSymlink, + outputPath: join(temporaryDirectory, 'submission.json'), + }), + ).rejects.toThrow('outside the repository'); + + const outputSymlink = join(temporaryDirectory, 'submission-link.json'); + await symlink(templatePath, outputSymlink); + await expect( + materializer.materializeSubmission({ + fixturesPath: externalFixturesPath, + outputPath: outputSymlink, + }), + ).rejects.toThrow('must not be a symlink'); + }); + + it('rejects unresolved placeholders, schema failures, and tool-profile drift', async () => { + const unresolvedPath = join(temporaryDirectory, 'unresolved.json'); + await copyFile(templatePath, unresolvedPath); + await expect( + preflight.preflightSubmission({ + fetchImpl: schemaFetch(), + sourceToolCatalog: matchingSourceToolCatalog(), + submissionPath: unresolvedPath, + }), + ).rejects.toThrow('still contains reviewer fixture placeholders'); + + const fixturesPath = join(temporaryDirectory, 'fixtures.json'); + const materializedPath = join(temporaryDirectory, 'materialized.json'); + await writeJson(fixturesPath, fixtures()); + await materializer.materializeSubmission({ fixturesPath, outputPath: materializedPath }); + + await expect( + preflight.preflightSubmission({ + fetchImpl: schemaFetch({ required: ['missing'] }), + sourceToolCatalog: matchingSourceToolCatalog(), + submissionPath: materializedPath, + }), + ).rejects.toThrow('does not match the current OpenAI schema'); + + await expect( + preflight.preflightSubmission({ + fetchImpl: schemaFetch(), + sourceToolCatalog: async () => new Map(), + submissionPath: materializedPath, + }), + ).rejects.toThrow('tool catalog differs from the OpenAI MCP profile'); + }); + + it('canonicalizes the preflight path before enforcing the repository boundary', async () => { + const submissionSymlink = join(temporaryDirectory, 'submission-link.json'); + await symlink(templatePath, submissionSymlink); + + await expect( + preflight.preflightSubmission({ + fetchImpl: schemaFetch(), + sourceToolCatalog: matchingSourceToolCatalog(), + submissionPath: submissionSymlink, + }), + ).rejects.toThrow('outside the repository'); + }); +});