From 556212be360c69eacc96a8973a42e3140452aae5 Mon Sep 17 00:00:00 2001 From: "J.Jason" <130959319+JJasonSun@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:37:03 +0800 Subject: [PATCH 1/2] docs: clarify embedding dimensions documentation and evidence scope --- references/examples.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/references/examples.md b/references/examples.md index fe21a56..bce77f2 100644 --- a/references/examples.md +++ b/references/examples.md @@ -16,9 +16,11 @@ an example edit alone does not refresh that evidence. The [ECNU embedding contract](https://developer.ecnu.edu.cn/vitepress/llm/api/embedding.html) accepts strings, not OpenAI token-ID arrays. Disable LangChain's token conversion. -Validate the returned length instead of sending a dimension-selection field: -the direct endpoint contract documents `model` and `input`, and a 1024-value -output, not selectable dimensions. Reject an empty list before making a call. +The official LangChain example sets `dimensions` to 1024, but the request table +lists only `model` and `input` and does not explain dimension selection. +This recipe omits `dimensions` and validates the documented 1024-value output, +following the dated recipe coverage above. That check does not establish whether +the service accepts or rejects the field. Reject an empty list before making a call. Standalone example; requires `langchain-openai`: From 4ad5b68f653b385e9b30c3539e3124b32275c378 Mon Sep 17 00:00:00 2001 From: "J.Jason" <130959319+JJasonSun@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:55:32 +0800 Subject: [PATCH 2/2] fix: align ECNU diagnostics with verified evidence --- scripts/smoke_test.py | 48 ++++++---------- scripts/validate_skill.py | 3 - tests/test_repository_contracts.py | 17 ++++++ tests/test_smoke_test.py | 91 ++++++++++++++++++++++-------- 4 files changed, 99 insertions(+), 60 deletions(-) diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py index 44bb754..420c327 100755 --- a/scripts/smoke_test.py +++ b/scripts/smoke_test.py @@ -1408,26 +1408,6 @@ def case_response_matches( return not bool(shape.get("reasoning_content_present")) if spec.case_id in {"responses_max_effort_low", "anthropic_effort_low"}: return bool(shape.get("reasoning_content_present")) - expected_anthropic_models = { - "anthropic_plus": "ecnu-plus", - "anthropic_max": "ecnu-max", - "anthropic_max_1m": "ecnu-max", - "anthropic_max_1m_fallback_plain_max": "ecnu-max", - } - if spec.case_id in expected_anthropic_models: - return shape.get("model") == expected_anthropic_models[spec.case_id] - expected_anthropic_aliases = { - "anthropic_sonnet_mapping": { - "claude-sonnet-4-20250514", - "ecnu-plus", - }, - "anthropic_opus_mapping": { - "claude-opus-4-1-20250805", - "ecnu-max", - }, - } - if spec.case_id in expected_anthropic_aliases: - return shape.get("model") in expected_anthropic_aliases[spec.case_id] return True @@ -2584,18 +2564,6 @@ def run_one(context: RunContext, spec: CaseSpec) -> dict[str, Any]: if executed.attempts > 1: shape["unexpected_retry_count"] = executed.attempts - 1 - authenticated_success = ( - spec.requires_valid_auth - and response.status is not None - and 200 <= response.status < 300 - and (spec.case_id != "models_valid" or bool(shape.get("model_ids"))) - ) - if authenticated_success: - context.state["valid_auth_observed"] = True - if spec.model: - successful_models = context.state.setdefault("successful_models", set()) - if isinstance(successful_models, set): - successful_models.add(spec.model) local_401_accepted = ( response.status == 401 and spec.case_id in CASE_LOCAL_401 @@ -2626,6 +2594,22 @@ def run_one(context: RunContext, spec: CaseSpec) -> dict[str, Any]: result = "pass" notes = ["structural check passed; generated content was omitted"] + # Later 401 exceptions require a successful protected control. + authenticated_success = ( + spec.requires_valid_auth + and spec.method == "POST" + and _official_api_endpoint(spec.endpoint) + and result == "pass" + and response.status is not None + and 200 <= response.status < 300 + ) + if authenticated_success: + context.state["valid_auth_observed"] = True + if spec.model: + successful_models = context.state.setdefault("successful_models", set()) + if isinstance(successful_models, set): + successful_models.add(spec.model) + if local_401_accepted: notes.append("case-specific 401 does not invalidate the already verified bearer token") diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 7c57fc5..2252ed3 100755 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -32,7 +32,6 @@ SECRET_RE = re.compile( r"(? list[str]: continue if SECRET_RE.search(content): errors.append(f"possible committed API key in {relative}") - if DIMENSION_ASSIGNMENT_RE.search(content): - errors.append(f"undocumented LangChain dimension request in {relative}") for line in find_literal_bearers(content): errors.append(f"literal Authorization bearer value in {relative}:{line}") if relative != Path("AGENTS.md"): diff --git a/tests/test_repository_contracts.py b/tests/test_repository_contracts.py index 199e372..8ba51d9 100644 --- a/tests/test_repository_contracts.py +++ b/tests/test_repository_contracts.py @@ -2,6 +2,7 @@ import json import re +import shutil import sys import tempfile import unittest @@ -99,6 +100,22 @@ def test_known_deviation_schema_and_status(self) -> None: class CurrentRepositoryContractsTest(unittest.TestCase): + def test_documentation_can_quote_embedding_dimensions(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "ecnu-api" + shutil.copytree( + ROOT, root, + ignore=shutil.ignore_patterns(".git", ".venv", "__pycache__", ".live-artifacts"), + ) + examples = root / "references/examples.md" + examples.write_text( + examples.read_text(encoding="utf-8") + + "\nThe official example uses `dimensions=1024`; acceptance is unverified.\n" + + "Do not send `dimensions=1024` in this recipe.\n", + encoding="utf-8", + ) + self.assertEqual(validate_skill.collect_errors(root), []) + def test_openai_tool_recipe_preserves_returned_fields(self) -> None: import httpx from openai import OpenAI diff --git a/tests/test_smoke_test.py b/tests/test_smoke_test.py index 075f565..3e7177e 100644 --- a/tests/test_smoke_test.py +++ b/tests/test_smoke_test.py @@ -521,6 +521,44 @@ def test_models_valid_rejects_empty_ids(self) -> None: ) ) + def test_authentication_requires_a_successful_protected_request(self) -> None: + discovery = {"object": "list", "data": [{"id": "ecnu-plus"}]} + chat = { + "choices": [{"message": {"role": "assistant", "content": "ok"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + for case_id, body, external, verified in ( + ("models_valid", discovery, False, False), + ("chat_basic_ecnu_plus", chat, False, True), + ("chat_basic_ecnu_plus", {}, False, False), + ("chat_basic_ecnu_plus", chat, True, False), + ): + with self.subTest(case=case_id, body=body, external=external): + control = self.case(case_id) + if external: + control = replace(control, endpoint="https://example.test/v1/chat/completions") + success = smoke_test.Execution(smoke_test.HttpResult( + 200, {"content-type": "application/json"}, json.dumps(body).encode() + ), "mock") + denied = smoke_test.Execution(smoke_test.HttpResult( + 401, {"content-type": "application/json"}, b'{"detail":"metadata failure"}' + ), "mock") + with smoke_test.temporary_artifacts() as directory: + context = smoke_test.RunContext( + "test-key", 1.0, directory, smoke_test.CreditBudget(10.0) + ) + with patch.object(smoke_test, "raw_executor", return_value=success): + smoke_test.run_one(context, control) + self.assertEqual(bool(context.state.get("valid_auth_observed")), verified) + self.assertEqual(bool(context.state.get("successful_models")), verified) + with patch.object(smoke_test, "raw_executor", return_value=denied): + record = smoke_test.run_one(context, self.case("error_unsupported_model")) + self.assertEqual(context.stop_reason is None, verified) + self.assertEqual( + any("already verified bearer" in note for note in record["notes"]), + verified, + ) + def test_expected_errors_require_json_and_tts_error_structure(self) -> None: generic = self.case("error_missing_model") text_shape = smoke_test.summarize_response( @@ -731,7 +769,7 @@ def test_compatibility_vision_is_unverified_and_behavior_checked(self) -> None: ) ) - def test_thinking_tool_and_anthropic_alias_invariants(self) -> None: + def test_thinking_tool_invariants(self) -> None: tool = self.case("chat_thinking_tool_first") shape = { "tool_call_count": 1, @@ -745,30 +783,33 @@ def test_thinking_tool_and_anthropic_alias_invariants(self) -> None: tool, 200, {**shape, "reasoning_content_present": True} ) ) - alias = self.case("anthropic_sonnet_mapping") - alias_shape = {"content_count": 1, "text_present": True, "model": "ecnu-max"} - self.assertFalse(smoke_test.case_response_matches(alias, 200, alias_shape)) - self.assertTrue( - smoke_test.case_response_matches( - alias, 200, {**alias_shape, "model": "claude-sonnet-4-20250514"} - ) - ) - self.assertTrue( - smoke_test.case_response_matches( - alias, 200, {**alias_shape, "model": "ecnu-plus"} - ) - ) - opus = self.case("anthropic_opus_mapping") - self.assertFalse( - smoke_test.case_response_matches( - opus, 200, {**alias_shape, "model": "ecnu-plus"} - ) - ) - self.assertTrue( - smoke_test.case_response_matches( - opus, 200, {**alias_shape, "model": "ecnu-max"} - ) - ) + + def test_anthropic_model_labels_are_diagnostic(self) -> None: + for case_id in ( + "anthropic_plus", "anthropic_max", "anthropic_max_1m", + "anthropic_max_1m_fallback_plain_max", + "anthropic_sonnet_mapping", "anthropic_opus_mapping", + ): + with self.subTest(case=case_id): + spec = self.case(case_id) + body = {"model": "backend-model-label", "stop_reason": "end_turn", + "content": [{"type": "text", "text": "ok"}]} + response = smoke_test.Execution(smoke_test.HttpResult( + 200, {"content-type": "application/json"}, json.dumps(body).encode() + ), "mock") + with smoke_test.temporary_artifacts() as directory: + context = smoke_test.RunContext( + "test-key", 1.0, directory, smoke_test.CreditBudget(10.0) + ) + with patch.object(smoke_test, "raw_executor", return_value=response): + record = smoke_test.run_one(context, spec) + self.assertEqual(record["result"], "pass") + shape = record["actual_response_shape"] + self.assertEqual(shape["model"], "backend-model-label") + self.assertFalse(shape["response_model_matches_request"]) + self.assertFalse(smoke_test.case_response_matches( + spec, 200, {**shape, "text_present": False} + )) def test_effort_invariants_and_anthropic_bearer_only(self) -> None: none = self.case("responses_max_effort_none")