From 440c9f9f74ae918d0d0f22db86adf9d77244e6da Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 4 Jun 2026 10:11:52 -0700 Subject: [PATCH] test: validate OpenClaw hook-only fallback exports Signed-off-by: mnajafian-nv --- crates/core/tests/unit/atif_tests.rs | 159 ++++++++++++++++++ .../tests/unit/observability/atof_tests.rs | 99 +++++++++++ .../unit/observability/openinference_tests.rs | 142 ++++++++++++++++ 3 files changed, 400 insertions(+) diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index 246b5291..5b9c858c 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -1042,6 +1042,165 @@ fn test_exporter_openclaw_timing_marks_become_system_steps_with_payloads() { ); } +#[test] +fn test_exporter_openclaw_hook_only_fallbacks_preserve_stripped_content_and_explicit_metrics() { + let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); + let stripped_uuid = Uuid::now_v7(); + let partial_uuid = Uuid::now_v7(); + let base = base_timestamp(); + + let mut stripped_start = event_builder(stripped_uuid, EventType::Start) + .name("openclaw-model-call") + .scope_type(ScopeType::Llm) + .input(json!({ + "headers": {}, + "content": { + "provider": "openai", + "model": "gpt-4", + "messages": [], + "imagesCount": 1, + "source": "openclaw.llm_output" + } + })) + .model_name("gpt-4") + .build(); + let mut stripped_end = event_builder(stripped_uuid, EventType::End) + .name("openclaw-model-call") + .scope_type(ScopeType::Llm) + .output(json!({ + "role": "assistant", + "assistant_texts_count": 1, + "usage": { + "cost_usd": 0.001 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + })) + .model_name("gpt-4") + .build(); + let mut partial_start = event_builder(partial_uuid, EventType::Start) + .name("openclaw-model-call") + .scope_type(ScopeType::Llm) + .input(json!({ + "headers": {}, + "content": { + "provider": "openai", + "model": "gpt-4", + "prompt": "visible prompt", + "messages": [{"role": "user", "content": "visible prompt"}], + "imagesCount": 0, + "source": "openclaw.llm_output" + } + })) + .model_name("gpt-4") + .build(); + let mut partial_end = event_builder(partial_uuid, EventType::End) + .name("openclaw-model-call") + .scope_type(ScopeType::Llm) + .output(json!({ + "role": "assistant", + "content": "visible answer", + "usage": { + "prompt_tokens": 42 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + })) + .model_name("gpt-4") + .build(); + + for (offset, event) in [ + &mut stripped_start, + &mut stripped_end, + &mut partial_start, + &mut partial_end, + ] + .into_iter() + .enumerate() + { + set_event_timestamp(event, base + chrono::Duration::milliseconds(offset as i64)); + } + + { + let mut state = exporter.state.lock().unwrap(); + state + .events + .extend([stripped_start, stripped_end, partial_start, partial_end]); + } + + let trajectory = exporter.export().unwrap(); + assert_atif_v17_shape(&trajectory); + assert_eq!(trajectory.steps.len(), 4); + + let stripped_user = &trajectory.steps[0]; + assert_eq!(stripped_user.source, "user"); + let stripped_user_message: serde_json::Value = + serde_json::from_str(stripped_user.message.as_str().unwrap()).unwrap(); + assert_eq!(stripped_user_message["provider"], json!("openai")); + assert_eq!(stripped_user_message["model"], json!("gpt-4")); + assert_eq!(stripped_user_message["messages"], json!([])); + assert_eq!(stripped_user_message["imagesCount"], json!(1)); + assert_eq!( + stripped_user_message["source"], + json!("openclaw.llm_output") + ); + assert!(stripped_user_message.get("prompt").is_none()); + assert!(stripped_user_message.get("systemPrompt").is_none()); + let stripped_user_extra: AtifStepExtra = + serde_json::from_value(stripped_user.extra.clone().unwrap()).unwrap(); + let stripped_request = stripped_user_extra.llm_request.unwrap(); + assert!(stripped_request.get("prompt").is_none()); + assert!(stripped_request.get("systemPrompt").is_none()); + assert_eq!(stripped_request["messages"], json!([])); + assert_eq!(stripped_request["imagesCount"], json!(1)); + + let stripped_agent = &trajectory.steps[1]; + assert_eq!(stripped_agent.source, "agent"); + let stripped_message: serde_json::Value = + serde_json::from_str(stripped_agent.message.as_str().unwrap()).unwrap(); + assert_eq!(stripped_message["assistant_texts_count"], json!(1)); + assert!(stripped_message.get("content").is_none()); + let stripped_metrics = stripped_agent.metrics.as_ref().unwrap(); + assert_eq!(stripped_metrics.prompt_tokens, None); + assert_eq!(stripped_metrics.completion_tokens, None); + assert_eq!(stripped_metrics.cached_tokens, None); + assert_eq!(stripped_metrics.cost_usd, Some(0.001)); + let stripped_agent_extra: AtifStepExtra = + serde_json::from_value(stripped_agent.extra.clone().unwrap()).unwrap(); + let stripped_response = stripped_agent_extra.llm_response.unwrap(); + assert!(stripped_response.get("content").is_none()); + assert_eq!(stripped_response["assistant_texts_count"], json!(1)); + + let partial_user = &trajectory.steps[2]; + assert_eq!(partial_user.source, "user"); + assert_eq!(partial_user.message, json!("visible prompt")); + let partial_user_extra: AtifStepExtra = + serde_json::from_value(partial_user.extra.clone().unwrap()).unwrap(); + let partial_request = partial_user_extra.llm_request.unwrap(); + assert_eq!(partial_request["prompt"], json!("visible prompt")); + assert_eq!( + partial_request["messages"][0]["content"], + json!("visible prompt") + ); + + let partial_agent = &trajectory.steps[3]; + assert_eq!(partial_agent.source, "agent"); + assert_eq!(partial_agent.message, json!("visible answer")); + let partial_metrics = partial_agent.metrics.as_ref().unwrap(); + assert_eq!(partial_metrics.prompt_tokens, Some(42)); + assert_eq!(partial_metrics.completion_tokens, None); + assert_eq!(partial_metrics.cached_tokens, None); + assert_eq!(partial_metrics.cost_usd, None); + + let final_metrics = trajectory.final_metrics.as_ref().unwrap(); + assert_eq!(final_metrics.total_prompt_tokens, Some(42)); + assert_eq!(final_metrics.total_completion_tokens, None); + assert_eq!(final_metrics.total_cached_tokens, None); + assert_eq!(final_metrics.total_cost_usd, Some(0.001)); +} + #[test] fn test_openai_responses_input_extracts_latest_user_content_block() { let message = extract_user_messages(&json!({ diff --git a/crates/core/tests/unit/observability/atof_tests.rs b/crates/core/tests/unit/observability/atof_tests.rs index c0bdcbb0..51585956 100644 --- a/crates/core/tests/unit/observability/atof_tests.rs +++ b/crates/core/tests/unit/observability/atof_tests.rs @@ -745,6 +745,105 @@ fn subscriber_preserves_openclaw_model_timing_marks_as_raw_jsonl() { assert_eq!(lines[1]["data"]["durationMs"], 42); } +#[test] +fn subscriber_preserves_openclaw_hook_only_fallback_payloads_as_raw_jsonl() { + let dir = temp_dir("atof-openclaw-hook-fallbacks"); + let exporter = AtofExporter::new( + AtofExporterConfig::new() + .with_output_directory(&dir) + .with_filename("events.jsonl"), + ) + .unwrap(); + let subscriber = exporter.subscriber(); + + let stripped_uuid = Uuid::now_v7(); + let partial_uuid = Uuid::now_v7(); + let parent_uuid = Uuid::now_v7(); + let events = [ + openclaw_replay_llm_event( + stripped_uuid, + Some(parent_uuid), + ScopeCategory::Start, + json!({ + "headers": {}, + "content": { + "provider": "openai", + "model": "gpt-4", + "messages": [], + "imagesCount": 1, + "source": "openclaw.llm_output" + } + }), + ), + openclaw_replay_llm_event( + stripped_uuid, + Some(parent_uuid), + ScopeCategory::End, + json!({ + "role": "assistant", + "assistant_texts_count": 1, + "usage": { + "cost_usd": 0.001 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + }), + ), + openclaw_replay_llm_event( + partial_uuid, + Some(parent_uuid), + ScopeCategory::Start, + json!({ + "headers": {}, + "content": { + "provider": "openai", + "model": "gpt-4", + "prompt": "visible prompt", + "messages": [{"role": "user", "content": "visible prompt"}], + "imagesCount": 0, + "source": "openclaw.llm_output" + } + }), + ), + openclaw_replay_llm_event( + partial_uuid, + Some(parent_uuid), + ScopeCategory::End, + json!({ + "role": "assistant", + "content": "visible answer", + "usage": { + "prompt_tokens": 42 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + }), + ), + ]; + + for event in &events { + subscriber(event); + } + exporter.force_flush().unwrap(); + + let lines = read_jsonl(exporter.path()); + assert_eq!(lines.len(), events.len()); + for (line, event) in lines.iter().zip(events.iter()) { + assert_eq!(line, &event.try_to_json_value().unwrap()); + assert_eq!(line["kind"], "scope"); + assert_eq!(line["parent_uuid"], parent_uuid.to_string()); + } + + assert!(lines[0]["data"]["content"].get("prompt").is_none()); + assert_eq!(lines[0]["data"]["content"]["messages"], json!([])); + assert!(lines[1]["data"].get("content").is_none()); + assert_eq!(lines[1]["data"]["usage"]["cost_usd"], 0.001); + assert_eq!(lines[3]["data"]["usage"]["prompt_tokens"], 42); + assert!(lines[3]["data"]["usage"].get("completion_tokens").is_none()); +} + #[test] fn register_deregister_flush_and_shutdown_work_with_runtime_events() { let _guard = crate::observability::test_mutex().lock().unwrap(); diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index c19ad65a..da710ec7 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -737,6 +737,148 @@ fn openclaw_model_timing_marks_attach_to_parent_spans() { assert!(!unpaired_attributes.contains_key("nemo_relay.mark.metadata_json")); } +#[test] +fn openclaw_hook_only_fallbacks_preserve_stripped_content_and_explicit_usage() { + let (provider, exporter) = make_provider(); + let mut processor = + OpenInferenceEventProcessor::new(provider.clone(), "test-scope".to_string()); + let stripped_uuid = Uuid::now_v7(); + let partial_uuid = Uuid::now_v7(); + + processor.process(&make_start_event( + stripped_uuid, + None, + "openclaw-model-call", + ScopeType::Llm, + Some(json!({ + "headers": {"authorization": "Bearer secret-token"}, + "content": { + "provider": "openai", + "model": "gpt-4", + "messages": [], + "imagesCount": 1, + "source": "openclaw.llm_output" + } + })), + )); + processor.process(&make_end_event( + stripped_uuid, + None, + "openclaw-model-call", + ScopeType::Llm, + Some(json!({ + "role": "assistant", + "assistant_texts_count": 1, + "usage": { + "cost_usd": 0.001 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + })), + )); + + processor.process(&make_start_event( + partial_uuid, + None, + "openclaw-model-call", + ScopeType::Llm, + Some(json!({ + "headers": {}, + "content": { + "provider": "openai", + "model": "gpt-4", + "prompt": "visible prompt", + "messages": [{"role": "user", "content": "visible prompt"}], + "imagesCount": 0, + "source": "openclaw.llm_output" + } + })), + )); + processor.process(&make_end_event( + partial_uuid, + None, + "openclaw-model-call", + ScopeType::Llm, + Some(json!({ + "role": "assistant", + "content": "visible answer", + "usage": { + "prompt_tokens": 42 + }, + "openclaw": { + "assistant_tool_call_names": [] + } + })), + )); + + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 2); + + let stripped_span = spans + .iter() + .find(|span| { + let attributes = attr_map(&span.attributes); + attributes.get("llm.cost.total") == Some(&"0.001".to_string()) + }) + .expect("missing stripped OpenClaw fallback span"); + let stripped_attributes = attr_map(&stripped_span.attributes); + assert_eq!( + stripped_attributes.get("input.mime_type"), + Some(&"application/json".to_string()) + ); + let stripped_input = stripped_attributes + .get("input.value") + .expect("missing stripped input.value"); + let parsed_input: serde_json::Value = serde_json::from_str(stripped_input).unwrap(); + assert_eq!(parsed_input["content"]["messages"], json!([])); + assert!(parsed_input["headers"].is_null() || parsed_input.get("headers").is_none()); + assert_eq!( + stripped_attributes.get("output.mime_type"), + Some(&"application/json".to_string()) + ); + let stripped_output = stripped_attributes + .get("output.value") + .expect("missing stripped output.value"); + let parsed_output: serde_json::Value = serde_json::from_str(stripped_output).unwrap(); + assert!(parsed_output.get("content").is_none()); + assert_eq!(parsed_output["assistant_texts_count"], json!(1)); + assert_eq!( + stripped_attributes.get("llm.cost.total"), + Some(&"0.001".to_string()) + ); + assert!(!stripped_attributes.contains_key("llm.token_count.prompt")); + assert!(!stripped_attributes.contains_key("llm.output_messages.0.message.content")); + assert!(!stripped_attributes.contains_key("llm.input_messages.0.message.role")); + assert_no_attr_contains(&stripped_attributes, "secret-token"); + + let partial_span = spans + .iter() + .find(|span| { + let attributes = attr_map(&span.attributes); + attributes.get("llm.token_count.prompt") == Some(&"42".to_string()) + }) + .expect("missing partial-usage OpenClaw fallback span"); + let partial_attributes = attr_map(&partial_span.attributes); + assert_eq!( + partial_attributes.get("input.value"), + Some(&"user: visible prompt".to_string()) + ); + assert_eq!( + partial_attributes.get("output.value"), + Some(&"visible answer".to_string()) + ); + assert_eq!( + partial_attributes.get("llm.token_count.prompt"), + Some(&"42".to_string()) + ); + assert!(!partial_attributes.contains_key("llm.token_count.completion")); + assert!(!partial_attributes.contains_key("llm.token_count.total")); + assert!(!partial_attributes.contains_key("llm.cost.total")); +} + #[test] fn llm_input_value_omits_request_headers() { let (provider, exporter) = make_provider();