diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index 38dd9222..30290863 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -3123,9 +3123,8 @@ limitations under the License. ## block-buffer - 0.10.4 **Repository URL**: https://github.com/RustCrypto/utils -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -3328,35 +3327,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -Copyright (c) 2018-2019 The RustCrypto Project Developers - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## block-buffer - 0.12.0 @@ -7927,9 +7898,8 @@ limitations under the License. ## crypto-common - 0.1.7 **Repository URL**: https://github.com/RustCrypto/traits -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -8132,35 +8102,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` -### License File: LICENSE-MIT -``` -Copyright (c) 2021 RustCrypto Developers - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## crypto-common - 0.2.1 @@ -8613,9 +8555,8 @@ SOFTWARE. ## digest - 0.10.7 **Repository URL**: https://github.com/RustCrypto/traits -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -8818,35 +8759,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -Copyright (c) 2017 Artyom Pavlov - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## digest - 0.11.2 @@ -13360,28 +13273,27 @@ limitations under the License. **Repository URL**: https://github.com/fizyk20/generic-array.git **License Type(s)**: MIT ### License: https://spdx.org/licenses/MIT.html -### License File: LICENSE ``` -The MIT License (MIT) - -Copyright (c) 2015 Bartłomiej Kamiński - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +The MIT License (MIT) + +Copyright (c) 2015 Bartłomiej Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` @@ -20430,9 +20342,8 @@ SOFTWARE. ## md-5 - 0.10.6 **Repository URL**: https://github.com/RustCrypto/hashes -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -20635,37 +20546,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` -### License File: LICENSE-MIT -``` -Copyright (c) 2006-2009 Graydon Hoare -Copyright (c) 2009-2013 Mozilla Foundation -Copyright (c) 2016 Artyom Pavlov - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## memchr - 2.8.0 @@ -38917,9 +38798,8 @@ limitations under the License. ## version_check - 0.9.5 **Repository URL**: https://github.com/SergioBenitez/version_check -**License Type(s)**: MIT/Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -39122,29 +39002,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -The MIT License (MIT) -Copyright (c) 2017-2018 Sergio Benitez -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` ## walkdir - 2.5.0 diff --git a/crates/cli/tests/coverage/server_tests.rs b/crates/cli/tests/coverage/server_tests.rs index 677a38e7..5581ce40 100644 --- a/crates/cli/tests/coverage/server_tests.rs +++ b/crates/cli/tests/coverage/server_tests.rs @@ -521,6 +521,444 @@ async fn serve_listener_hermes_api_hooks_write_atof_category_profile_and_fidelit assert_eq!(lossy_start["data"]["content"]["message_count"], json!(2)); } +#[tokio::test] +async fn serve_listener_hermes_api_request_error_writes_lossy_atof_error_event() { + let _guard = PLUGIN_TEST_LOCK.lock().await; + let _ = nemo_relay::plugin::clear_plugin_configuration(); + + let temp = tempfile::tempdir().unwrap(); + let atof_dir = temp.path().join("atof"); + std::fs::create_dir_all(&atof_dir).unwrap(); + let mut config = test_config(); + config.plugin_config = Some(json!({ + "version": 1, + "components": [ + { + "kind": "observability", + "enabled": true, + "config": { + "version": 1, + "atof": { + "enabled": true, + "output_directory": atof_dir, + "filename": "events.jsonl", + "mode": "overwrite" + } + } + } + ] + })); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let url = format!("http://{address}"); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handle = + tokio::spawn(async move { serve_listener(listener, config, Some(shutdown_rx)).await }); + + wait_for_gateway(&url).await; + let client = test_http_client(); + + let response = client + .post(format!("{url}/hooks/hermes")) + .json(&json!({ + "hook_event_name": "pre_api_request", + "session_id": "hermes-atof-error", + "extra": { + "task_id": "task-err", + "api_request_id": "turn-1:api:3", + "api_call_count": 3, + "model": "qwen", + "provider": "custom", + "request": { + "method": "POST", + "body": { + "model": "qwen", + "messages": [ + { "role": "user", "content": "hello" } + ] + } + } + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .post(format!("{url}/hooks/hermes")) + .json(&json!({ + "hook_event_name": "api_request_error", + "session_id": "hermes-atof-error", + "extra": { + "task_id": "task-err", + "api_request_id": "turn-1:api:3", + "api_call_count": 3, + "model": "qwen", + "provider": "custom", + "status_code": 502, + "retry_count": 1, + "max_retries": 2, + "retryable": true, + "reason": "upstream", + "error": { + "type": "BadGateway", + "message": "gateway upstream error" + } + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + shutdown_tx.send(()).unwrap(); + handle.await.unwrap().unwrap(); + + let events = std::fs::read_to_string(temp.path().join("atof/events.jsonl")).unwrap(); + let llm_events = events + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["category"] == "llm") + .collect::>(); + assert_eq!( + llm_events.len(), + 2, + "expected Hermes error-path LLM exports, got {llm_events:?}" + ); + + let end = llm_events + .iter() + .find(|event| { + event["scope_category"] == "end" + && event["metadata"]["api_call_id"] == json!("turn-1:api:3") + }) + .unwrap(); + let start = llm_events + .iter() + .find(|event| { + event["scope_category"] == "start" + && event["metadata"]["api_call_id"] == json!("turn-1:api:3") + }) + .unwrap(); + assert_eq!(start["metadata"]["provider_payload_exact"], json!(true)); + assert_eq!( + start["metadata"]["fidelity_source"], + json!("hermes_api_hooks_sanitized") + ); + assert_eq!( + start["data"]["content"]["messages"][0]["content"], + json!("hello") + ); + assert_eq!(end["category_profile"]["model_name"], json!("qwen")); + assert_eq!(end["metadata"]["provider_payload_exact"], json!(false)); + assert_eq!( + end["metadata"]["fidelity_source"], + json!("hermes_api_hooks") + ); + assert_eq!(end["data"]["status_code"], json!(502)); + assert_eq!(end["data"]["retry_count"], json!(1)); + assert_eq!(end["data"]["retryable"], json!(true)); + assert_eq!(end["data"]["reason"], json!("upstream")); + assert_eq!( + end["data"]["error"]["message"], + json!("gateway upstream error") + ); +} + +#[tokio::test] +async fn serve_listener_routed_gateway_wire_formats_write_atof_category_profile_and_usage() { + let _guard = PLUGIN_TEST_LOCK.lock().await; + let _ = nemo_relay::plugin::clear_plugin_configuration(); + + async fn anthropic_messages() -> TestServer { + async fn messages(_headers: HeaderMap, _request: Request) -> impl IntoResponse { + Json(json!({ + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4", + "content": [ + {"type": "text", "text": "I will search."}, + {"type": "tool_use", "id": "toolu_01", "name": "search", "input": {"query": "file"}} + ], + "stop_reason": "tool_use", + "usage": { + "input_tokens": 11, + "output_tokens": 7, + "cache_read_input_tokens": 3, + "cost": {"total": 0.0042} + } + })) + } + + let app = Router::new().route("/v1/messages", post(messages)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + TestServer { + url: format!("http://{address}"), + handle, + } + } + + async fn openai_routed() -> TestServer { + async fn chat(_headers: HeaderMap, request: Request) -> impl IntoResponse { + let path = request.uri().path().to_string(); + if path == "/v1/responses" { + Json(json!({ + "id": "resp_1", + "status": "completed", + "output": [ + { + "type": "message", + "content": [{"type": "output_text", "text": "I will check the weather."}] + }, + { + "type": "function_call", + "call_id": "call_weather_1", + "name": "get_weather", + "arguments": "{\"city\":\"SF\"}", + "status": "completed" + } + ], + "usage": { + "input_tokens": 75, + "output_tokens": 20, + "total_tokens": 95, + "input_tokens_details": {"cached_tokens": 10}, + "cost_usd": 0.005 + } + })) + } else { + Json(json!({ + "choices": [{ + "message": { + "role": "assistant", + "content": "I will inspect.", + "tool_calls": [ + { + "id": "call_read_1", + "type": "function", + "function": {"name": "read", "arguments": "{\"path\":\"api.py\"}"} + } + ] + }, + "finish_reason": "tool_calls" + }], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 4, + "total_tokens": 7, + "prompt_tokens_details": {"cached_tokens": 2}, + "cost_usd": 0.001 + } + })) + } + } + + let app = Router::new() + .route("/v1/chat/completions", post(chat)) + .route("/v1/responses", post(chat)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + TestServer { + url: format!("http://{address}"), + handle, + } + } + + let temp = tempfile::tempdir().unwrap(); + let atof_dir = temp.path().join("atof"); + std::fs::create_dir_all(&atof_dir).unwrap(); + + let anthropic_upstream = anthropic_messages().await; + let openai_upstream = openai_routed().await; + + let mut config = test_config(); + config.anthropic_base_url = anthropic_upstream.url(); + config.openai_base_url = openai_upstream.url(); + config.plugin_config = Some(json!({ + "version": 1, + "components": [ + { + "kind": "observability", + "enabled": true, + "config": { + "version": 1, + "atof": { + "enabled": true, + "output_directory": atof_dir, + "filename": "events.jsonl", + "mode": "overwrite" + } + } + } + ] + })); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let url = format!("http://{address}"); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handle = + tokio::spawn(async move { serve_listener(listener, config, Some(shutdown_rx)).await }); + + wait_for_gateway(&url).await; + let client = test_http_client(); + + let response = client + .post(format!("{url}/v1/messages")) + .header("content-type", "application/json") + .header("x-api-key", "sk-ant-test") + .header("x-nemo-relay-session-id", "hermes-routed-atof") + .json(&json!({ + "model": "claude-sonnet-4", + "messages": [{"role": "user", "content": "Find the file."}], + "tools": [{"name": "search", "input_schema": {"type": "object"}}] + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .post(format!("{url}/v1/responses")) + .header("content-type", "application/json") + .header("authorization", "Bearer test") + .header("x-nemo-relay-session-id", "hermes-routed-atof") + .json(&json!({ + "model": "gpt-4o", + "input": "Find the weather.", + "tools": [{"type": "function", "name": "get_weather"}] + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .post(format!("{url}/v1/chat/completions")) + .header("content-type", "application/json") + .header("authorization", "Bearer test") + .header("x-nemo-relay-session-id", "hermes-routed-atof") + .json(&json!({ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Inspect the files."}], + "tools": [{"type": "function", "function": {"name": "read"}}] + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + shutdown_tx.send(()).unwrap(); + handle.await.unwrap().unwrap(); + + let events = std::fs::read_to_string(temp.path().join("atof/events.jsonl")).unwrap(); + let llm_events = events + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["category"] == "llm") + .collect::>(); + assert_eq!( + llm_events.len(), + 6, + "expected three routed LLM start/end pairs, got {llm_events:?}" + ); + + let anthropic_start = llm_events + .iter() + .find(|event| { + event["scope_category"] == "start" + && event["name"] == "anthropic.messages" + && event["metadata"]["gateway_path"] == "/v1/messages" + }) + .unwrap(); + assert_eq!( + anthropic_start["category_profile"]["model_name"], + json!("claude-sonnet-4") + ); + assert_eq!( + anthropic_start["data"]["content"]["messages"][0]["content"], + json!("Find the file.") + ); + + let anthropic_end = llm_events + .iter() + .find(|event| { + event["scope_category"] == "end" + && event["name"] == "anthropic.messages" + && event["metadata"]["gateway_path"] == "/v1/messages" + }) + .unwrap(); + assert_eq!( + anthropic_end["category_profile"]["annotated_response"]["tool_calls"][0]["id"], + json!("toolu_01") + ); + assert_eq!(anthropic_end["data"]["content"][1]["id"], json!("toolu_01")); + assert_eq!(anthropic_end["data"]["usage"]["input_tokens"], json!(11)); + assert_eq!( + anthropic_end["data"]["usage"]["cost"]["total"], + json!(0.0042) + ); + + let responses_end = llm_events + .iter() + .find(|event| { + event["scope_category"] == "end" + && event["name"] == "openai.responses" + && event["metadata"]["gateway_path"] == "/v1/responses" + }) + .unwrap(); + assert_eq!( + responses_end["category_profile"]["model_name"], + json!("gpt-4o") + ); + assert_eq!( + responses_end["category_profile"]["annotated_response"]["tool_calls"][0]["id"], + json!("call_weather_1") + ); + assert_eq!( + responses_end["data"]["output"][1]["call_id"], + json!("call_weather_1") + ); + assert_eq!( + responses_end["data"]["usage"]["input_tokens_details"]["cached_tokens"], + json!(10) + ); + assert_eq!(responses_end["data"]["usage"]["cost_usd"], json!(0.005)); + + let chat_end = llm_events + .iter() + .find(|event| { + event["scope_category"] == "end" + && event["name"] == "openai.chat_completions" + && event["metadata"]["gateway_path"] == "/v1/chat/completions" + }) + .unwrap(); + assert_eq!(chat_end["category_profile"]["model_name"], json!("gpt-4o")); + assert_eq!( + chat_end["category_profile"]["annotated_response"]["tool_calls"][0]["id"], + json!("call_read_1") + ); + assert_eq!( + chat_end["data"]["choices"][0]["message"]["tool_calls"][0]["id"], + json!("call_read_1") + ); + assert_eq!( + chat_end["data"]["usage"]["prompt_tokens_details"]["cached_tokens"], + json!(2) + ); + assert_eq!(chat_end["data"]["usage"]["cost_usd"], json!(0.001)); +} + #[tokio::test] async fn serve_listener_activates_any_registered_plugin_kind() { let _guard = PLUGIN_TEST_LOCK.lock().await; diff --git a/crates/cli/tests/coverage/session_tests.rs b/crates/cli/tests/coverage/session_tests.rs index 07c76214..7689afaa 100644 --- a/crates/cli/tests/coverage/session_tests.rs +++ b/crates/cli/tests/coverage/session_tests.rs @@ -1562,6 +1562,128 @@ async fn hermes_exact_api_hooks_write_atif_request_response_and_cost() { })); } +#[tokio::test] +async fn hermes_api_request_error_writes_atif_error_step_and_fidelity() { + let _guard = OBSERVABILITY_PLUGIN_TEST_LOCK.lock().await; + let temp = tempfile::tempdir().unwrap(); + let atif_dir = temp.path().join("atif"); + install_test_atif_plugin(&atif_dir).await; + let config = session_test_config(); + let manager = SessionManager::new(config); + let headers = HeaderMap::new(); + + for payload in [ + json!({ + "hook_event_name": "on_session_start", + "session_id": "hermes-error" + }), + json!({ + "hook_event_name": "pre_api_request", + "session_id": "hermes-error", + "extra": { + "task_id": "task-err", + "api_request_id": "turn-1:api:3", + "api_call_count": 3, + "provider": "custom", + "model": "qwen", + "request": { + "method": "POST", + "body": { + "model": "qwen", + "messages": [ + { "role": "user", "content": "hello" } + ] + } + } + } + }), + json!({ + "hook_event_name": "api_request_error", + "session_id": "hermes-error", + "extra": { + "task_id": "task-err", + "api_request_id": "turn-1:api:3", + "api_call_count": 3, + "provider": "custom", + "model": "qwen", + "status_code": 502, + "retry_count": 1, + "max_retries": 2, + "retryable": true, + "reason": "upstream", + "error": { + "type": "BadGateway", + "message": "gateway upstream error" + } + } + }), + json!({ + "hook_event_name": "on_session_finalize", + "session_id": "hermes-error" + }), + ] { + let outcome = crate::adapters::hermes::adapt(payload, &headers); + manager + .apply_events(&headers, outcome.events) + .await + .unwrap(); + } + + clear_plugin_configuration().unwrap(); + let atif = read_atif_for_session(&atif_dir, "hermes-error"); + let steps = atif["steps"].as_array().unwrap(); + assert_eq!(steps.len(), 2); + assert_eq!(steps[0]["message"], json!("hello")); + assert_eq!(steps[1]["source"], json!("agent")); + assert_eq!(steps[1]["extra"]["llm_response"]["status_code"], json!(502)); + assert_eq!(steps[1]["extra"]["llm_response"]["retry_count"], json!(1)); + assert_eq!(steps[1]["extra"]["llm_response"]["retryable"], json!(true)); + assert_eq!( + steps[1]["extra"]["llm_response"]["reason"], + json!("upstream") + ); + assert_eq!( + steps[1]["extra"]["llm_response"]["error"]["message"], + json!("gateway upstream error") + ); + let observed_events = atif["extra"]["observed_events"].as_array().unwrap(); + assert!( + observed_events.len() >= 4, + "expected Hermes error trajectory to keep observed events, got {}", + serde_json::to_string_pretty(&atif["extra"]["observed_events"]).unwrap() + ); + let error_event = observed_events + .iter() + .find(|event| { + event["scope_category"] == json!("end") + && event["metadata"]["api_call_id"] == json!("turn-1:api:3") + }) + .unwrap(); + let request_event = observed_events + .iter() + .find(|event| { + event["scope_category"] == json!("start") + && event["metadata"]["api_call_id"] == json!("turn-1:api:3") + }) + .unwrap(); + assert_eq!( + request_event["metadata"]["provider_payload_exact"], + json!(true) + ); + assert_eq!( + request_event["metadata"]["fidelity_source"], + json!("hermes_api_hooks_sanitized") + ); + assert_eq!( + error_event["metadata"]["provider_payload_exact"], + json!(false) + ); + assert_eq!( + error_event["metadata"]["fidelity_source"], + json!("hermes_api_hooks") + ); +} + #[tokio::test] async fn hermes_lossy_api_hooks_write_atif_fidelity_markers() { let _guard = OBSERVABILITY_PLUGIN_TEST_LOCK.lock().await; diff --git a/crates/core/src/observability/openinference.rs b/crates/core/src/observability/openinference.rs index deb1b55c..092ed669 100644 --- a/crates/core/src/observability/openinference.rs +++ b/crates/core/src/observability/openinference.rs @@ -647,6 +647,11 @@ fn start_attributes(event: &Event) -> Vec { let is_llm = event .category() .is_some_and(|category| category.as_str() == "llm"); + if is_llm { + // Final span metadata should reflect the completed event, especially for mixed-fidelity + // Hermes flows where the request can be exact but the terminal error is lossy. + attributes.retain(|attribute| attribute.key.as_str() != oi::METADATA.as_str()); + } let handle_attributes = event.attributes(); if handle_attributes.is_some_and(|attributes| !attributes.is_empty()) { push_serialized( @@ -700,6 +705,10 @@ fn end_attributes(event: &Event) -> Vec { .category() .is_some_and(|category| category.as_str() == "llm"); + if let Some(metadata) = event.metadata().and_then(to_json_string) { + attributes.push(KeyValue::new(oi::METADATA, metadata)); + } + push_serialized( &mut attributes, "nemo_relay.end.output_json", diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index da710ec7..4488b4b9 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -2486,6 +2486,116 @@ fn hermes_exact_api_payloads_emit_openinference_text_usage_and_metadata() { ); } +#[test] +fn hermes_api_request_error_emits_openinference_json_output_and_metadata() { + let (provider, exporter) = make_provider(); + let mut processor = + OpenInferenceEventProcessor::new(provider.clone(), "test-scope".to_string()); + let uuid = Uuid::now_v7(); + let start_metadata = json!({ + "provider_payload_exact": true, + "fidelity_source": "hermes_api_hooks_sanitized" + }); + let end_metadata = json!({ + "provider_payload_exact": false, + "fidelity_source": "hermes_api_hooks" + }); + + processor.process(&Event::Scope(ScopeEvent::new( + BaseEvent::builder() + .uuid(uuid) + .name("custom") + .data(json!({ + "model": "qwen", + "messages": [{ "role": "user", "content": "hello" }] + })) + .metadata(start_metadata) + .build(), + ScopeCategory::Start, + Vec::new(), + EventCategory::llm(), + Some(CategoryProfile::builder().model_name("qwen").build()), + ))); + processor.process(&Event::Scope(ScopeEvent::new( + BaseEvent::builder() + .uuid(uuid) + .name("custom") + .data(json!({ + "status_code": 502, + "retry_count": 1, + "max_retries": 2, + "retryable": true, + "reason": "upstream", + "error": { + "type": "BadGateway", + "message": "gateway upstream error" + } + })) + .metadata(end_metadata) + .build(), + ScopeCategory::End, + Vec::new(), + EventCategory::llm(), + Some(CategoryProfile::builder().model_name("qwen").build()), + ))); + + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + let attributes = attr_map(&spans[0].attributes); + assert_eq!( + attributes.get("openinference.span.kind"), + Some(&"LLM".to_string()) + ); + assert_eq!(attributes.get("llm.model_name"), Some(&"qwen".to_string())); + assert_eq!( + attributes.get("input.value"), + Some(&"user: hello".to_string()) + ); + assert_eq!( + serde_json::from_str::(attributes.get("output.value").unwrap()).unwrap(), + json!({ + "status_code": 502, + "retry_count": 1, + "max_retries": 2, + "retryable": true, + "reason": "upstream", + "error": { + "type": "BadGateway", + "message": "gateway upstream error" + } + }) + ); + assert_eq!( + attributes.get("output.mime_type"), + Some(&"application/json".to_string()) + ); + assert_eq!( + serde_json::from_str::( + attributes.get("nemo_relay.end.output_json").unwrap(), + ) + .unwrap(), + json!({ + "status_code": 502, + "retry_count": 1, + "max_retries": 2, + "retryable": true, + "reason": "upstream", + "error": { + "type": "BadGateway", + "message": "gateway upstream error" + } + }) + ); + assert_attr_contains(&attributes, "metadata", "\"provider_payload_exact\":false"); + assert_attr_contains( + &attributes, + "metadata", + "\"fidelity_source\":\"hermes_api_hooks\"", + ); +} + #[test] fn llm_end_with_inconsistent_manual_usage_omits_invalid_total_tokens() { let (provider, exporter) = make_provider();