From c994d251db7debe3abc489fa36b91d7672982530 Mon Sep 17 00:00:00 2001 From: ajayr4j Date: Mon, 3 Aug 2026 10:29:08 +0530 Subject: [PATCH] Add leaderboard entry: Trinity / DeepSeek V4 Pro (78.6% accuracy) Trinity (11-skill Claude Code skill pack) + DeepSeek V4 Pro (via High-Flyer, routed through litellm), full 140-trial run across all 14 Enterprise-Bench l1-l2-bench tasks (10 trials each). - Overall accuracy: 78.57% (110/140 successful trials) - 2 genuine non-infra errors (not counted against accuracy as N/A, scored 0): eng-l1-c__MjQfY3J (DeepSeek V4 Pro thinking-mode/reasoning_content litellm compatibility bug) and eng-l2-b__GjkKAcC (transient upstream billing/balance exhaustion, confirmed resolved mid-run) - Cost is displayed as "-" (total_cost_usd: 0.0): this run routed through litellm to a non-Anthropic model, and Claude Code's stream-json output did not populate trustworthy usage/cost stats -- the same instrumentation gap already disclosed in PR #11 (Claude Code / DeepSeek V4 Pro bare harness) and PR #9 (GLM-5.2). An independent tiktoken cl100k reconstruction from the raw transcripts counts ~4.33M input tokens vs. the ~42.4M self-reported here, a ~10x discrepancy, so the self-reported cost figure is not used. Source job (public): https://hub.harborframework.com/jobs/02b0a6e5-ce65-4a5e-9494-feab0f5f5ed4 --- .../2026-08-03__trinity__deepseek-v4-pro.yaml | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 leaderboard/entries/2026-08-03__trinity__deepseek-v4-pro.yaml diff --git a/leaderboard/entries/2026-08-03__trinity__deepseek-v4-pro.yaml b/leaderboard/entries/2026-08-03__trinity__deepseek-v4-pro.yaml new file mode 100644 index 0000000..8ee186d --- /dev/null +++ b/leaderboard/entries/2026-08-03__trinity__deepseek-v4-pro.yaml @@ -0,0 +1,168 @@ +# Trinity / DeepSeek V4 Pro +# Source job: 02b0a6e5-ce65-4a5e-9494-feab0f5f5ed4 +# https://hub.harborframework.com/jobs/02b0a6e5-ce65-4a5e-9494-feab0f5f5ed4 +rows: +- metadata: + agent_display_name: Trinity + model_display_name: DeepSeek V4 Pro + agent_org_display_name: Ajay Raj S + model_org_display_name: High-Flyer + metrics: + accuracy: 78.57 + display_accuracy: 78.6% + uncached_input_tokens: 42386558 + cached_input_tokens: 0 + output_tokens: 737085 + total_tokens: 43123643 + avg_trial_duration_sec: 240.97 + pass_at_2: 0.8937 + pass_at_3: 0.9375 + pass_at_4: 0.9622 + pass_at_5: 0.9779 + pass_at_8: 0.9984 + pass_at_10: 1.0000 + n_trials: 140 + total_cost_usd: 0.0 + display_total_cost_usd: "-" + status: display + trial_ids: + - ad60e8ac-9625-4972-bb81-4c3151ea2c3f + - 9c47dcf4-33d5-45cb-8cd5-cae0c5610f01 + - 546ae17e-670e-4b86-ba8d-dab9352a0d7d + - 3088ef33-0bb2-4b1c-bc39-a218c69becbc + - 2ee179b3-cca7-473f-bbcc-db34dd9d1f0b + - 021431e8-e013-44f5-baae-2821a59735ef + - 37229bc3-c05c-4948-8998-ce1076b14af3 + - c80e5170-cd98-40f1-9ac8-97040c6f8c64 + - ece131fb-fbde-4065-94c3-ec9142e55b97 + - 53843a5c-b39c-4782-b984-83a4083f9495 + - f2b7234d-68ae-4f9d-a7ec-17562d1626f9 + - b36822b0-c511-4a0a-ba67-a0a71813662f + - ae9b4256-7bce-4a88-b9e3-fc1d363f8343 + - a99ac075-6506-4506-a035-0a30aa3e8572 + - 15c46c91-a552-4165-83c8-9254d8df5d9d + - 2ca98a17-8547-4df1-aeda-fb50f9052ffb + - 8ff41c97-2fef-4075-8871-78e3da086b0c + - 3be4a838-0d93-464c-9108-ec59a07911f9 + - a2897668-9a76-4392-84c9-eb5fb9e7fb40 + - de75982d-9769-4169-88ed-cb8711f12366 + - b395a47d-b106-478e-af79-f25eaa4a72cd + - 132bd60c-b874-46c0-bfec-272f8a539eba + - a29084d8-17e4-4466-8f32-e84be0fb0cfa + - d8edb8f0-8194-42e7-a30e-59f9129a79d4 + - 384e76fe-6c93-46a9-9033-e994f3292e08 + - 89b0ead3-fc30-4070-82c9-a79c8fef5dfc + - 2792cfb9-b1ce-483e-87bf-6606c1cdc0e4 + - 5bbd3857-343b-497e-9836-65b38cbf38b7 + - 955d6a68-2ad3-4f99-93b4-ed84a6f1073a + - 5013e349-4e3c-4765-a31e-6a42656d1c97 + - 2f3363cc-9b42-499e-ba1b-61086ab96bc5 + - 504d733a-0dee-4c26-9d91-e39795b5d214 + - ffbbc4a2-3289-46f7-8cc3-306b14bd7375 + - 64493f3d-7984-4a2d-9dfa-6b7579bb19f7 + - 53fb34bf-037f-446a-a802-c08c03cd9dc0 + - 87fc8f1b-96d2-4fda-b163-89afbae1badb + - 2d761cbe-d5ab-43fa-a988-793630ad4888 + - aa763010-955b-43c6-8bd2-a1d95c7b2e12 + - 5a057105-9de8-4592-ba82-051a550a23a2 + - b50c8e8a-577a-4ce6-a7c7-000a00bd8d5f + - 85f30f73-cf3f-4007-8051-b6d8985c44ad + - 9d75cf11-e07d-4f54-a3fc-d2b270ff60b8 + - 085bdbf4-4c77-461c-9dc8-92b9bcf7bbb6 + - a593ed45-c932-4ab0-b7d4-2ce49dba9ea4 + - b99bd42d-c0e6-487e-b42c-1a4dd8405ef3 + - b13a37f6-3346-4a0a-a0bc-b9dc696fbabd + - 3e042638-f6ee-4fbf-ac9f-bd466a1bb0f7 + - 88c264e4-ffe2-4222-97a3-029201470dcb + - e577a514-7a7b-47d4-8804-1e0d99182e24 + - 42a9d952-a441-4dc8-a9e3-1527d039de28 + - 88e9503a-ddbc-46ab-9016-45fd510485d6 + - 64a41cab-d38d-4284-89a9-fdd095caaf0c + - a6dfe201-fe4f-4595-9d7b-84b916c15b19 + - 4568ef13-5f8d-4f59-8a21-8fa47c8b0e8d + - 3ad37f15-f3cb-4968-b118-34cb42bc064d + - 120a825c-4f32-4add-9ef6-41b05611fba1 + - 46862dc5-6ec9-4ade-9ec8-d1d148b4e0f5 + - c9222076-892d-4612-ac14-7a098d534a86 + - 316867a8-b6f3-40e9-b087-5ef7653f7fa3 + - aee425e9-0667-46a9-a863-a61e33a952c6 + - 70465574-77d0-4508-b93d-aa6048c6a115 + - b3424af8-39b7-42b3-90b1-4a64f24f7b6e + - e2ecedd0-cc19-4590-ab4e-f297cb9287c7 + - 8a1d8540-13e9-4ca0-9da2-0abfa9c0adeb + - b79184cd-7b0c-4484-ac34-c71cdd7c6c64 + - fc97e9cf-d2a7-4a3c-9cd4-3ce1d6d3d44f + - 0d088b5b-bd7f-4ac4-b91d-5b13446b9611 + - e7ce3d0f-9adc-4387-ad50-a95f42433f9d + - 1ca97b64-ab34-4d35-811d-72f54dc764fc + - c1e6ce94-9c7b-4285-be89-6f73e58145a2 + - b1b3b78b-7e89-4679-a25d-22ff8dcc13e3 + - 52c34363-1d65-4b32-9bb0-4ef22a71cae9 + - 0c925d12-a64d-4258-8958-dd4bc1a9055a + - a83b7d1c-b310-43ca-8a28-518617541270 + - 241ae5c0-e564-4fc6-8b06-23fb8c948329 + - e53d06b4-519c-4865-9086-45deaabe0da3 + - a93711b6-2293-4d42-99d4-fe35e2d76715 + - 7a084e23-e124-4e43-afb2-ac716fbf0e62 + - 223432c6-6e34-4445-b417-5ab3a68982f7 + - b922b6ab-f8b8-49c0-9b79-d619a012332f + - 1501fb66-fd36-4d1b-80c5-f0fbf0139fd7 + - 3f36c59f-b458-496d-93d7-223b7454b02e + - 69782205-d3dd-4067-ad79-fe275ce2ee85 + - def9d4c2-2a38-4e8e-895f-90a95e4ddd24 + - 198757c8-3418-44dd-8e05-bf0fe45a1f1d + - f386f5c7-a98f-4ee5-b37f-723b8063d99f + - 4b7875c4-a634-42b2-9f52-db0ac3eeb23c + - 2e20eff8-d2ff-4e1c-bd9a-bbf6d4762be9 + - d263d1f2-e339-4828-aea0-29397eef977f + - 99b7a0b6-be77-4f89-8319-5ea1ddb357e9 + - b143abce-5ec9-45ac-a03d-eb04903dd39e + - 6481dc64-b180-4dff-ab41-55b13badac71 + - b9660dd1-8e33-4910-83b0-a9eba9d77a6e + - ee1f2bfc-f263-4f9e-b7f3-ae1699d2e565 + - a5452343-099a-4ccd-96fe-dfcfd74d89b2 + - 0657694e-52a5-4e31-bc62-61d0ac3a9322 + - 7b4f985e-a50b-4c0f-9a48-f7169e40fd40 + - afadc820-25ef-46ee-9d84-bc3b6c8b1105 + - 49f934d2-15ec-43c6-9942-c191edb820e1 + - 57063b97-c1ce-490c-9f30-a68db596226f + - 632edb8e-7a0e-45f2-9d9d-a40f61da7493 + - 28780a67-c429-4fb8-b57d-fce00202fe91 + - 959113dc-6e5b-4109-8321-8d076dccebe5 + - eb84fd69-b12e-4e84-be14-b909fc9a9dc9 + - f3cc4c82-ee66-44c5-a235-9236ef3b174b + - 0193f2d2-07c1-4ddd-902a-2e8fdef0eb6e + - 3231c070-b6fd-40ba-8b2c-872d740631d0 + - ed92ec77-ab29-4b22-8b9b-9fb7fe0f87d0 + - 4a0c524f-8597-4988-92fa-a1c5830925c0 + - 83b032f6-bc0f-4f75-be38-382f3ae1d190 + - b8c7f9b1-3a08-4168-89db-a690f030af43 + - bd942498-3421-442f-b3d8-97e47464f795 + - af1a8042-64b2-4e73-9317-a8f2c5869f22 + - d2af61bd-472c-4557-ba83-2c6dc3b92c2c + - c5bc7ad6-30bb-4107-8f36-431fbd9d15b4 + - b1db6130-d3c0-440b-b7e9-76aa2b51ee85 + - 0965d20f-d135-48dc-b034-b0e65504b4df + - 8a13e3a4-839e-406c-9439-aa8bd5043fca + - 88365a08-a4e0-4657-838f-9b392ee4f316 + - fd5ebbb1-7aeb-40b9-aed5-835fef3ff3a1 + - 390fff1a-1c4d-40f7-bb60-6f6a2c3f4c7a + - 41c971e3-40cf-43c1-9f64-db8b144c1fdc + - 3cbbc572-d04d-4a33-b487-c53ab7dc9813 + - ebe4664b-8938-436b-a6f7-320f408e5448 + - 901eff04-25bf-4983-aaeb-a0a346b3d2c1 + - 96f6a1c4-4a1d-44c5-95d3-bb88768bbde5 + - 5df633a3-3b53-412d-81e9-a60f84a4548b + - 5ccb5ced-3fb4-4bcb-a080-f62d214cf3d3 + - d1032f8d-ed7a-4673-bd32-71b71e14e563 + - d1692695-0a27-4e39-8302-ce6714d9cf10 + - 1d31d053-003f-4561-91f5-eb6e678f3dda + - b709bacb-baed-42e6-9553-eb4cdbfb6ed7 + - b2c41f14-efa5-4bbe-9c98-6b15fbde9e75 + - a92fe40a-0f71-45e3-80b8-ed57f0bc9aa5 + - 29c62724-2825-4388-8e87-96b5011dfe59 + - 360e490d-c672-40c0-99a6-65526523d6f4 + - bf8a48e6-c28e-4e95-a695-8a845bc3f635 + - fbb11e68-a7b6-42bc-984f-bfb47e2a623b + - 49298029-5166-4c9e-a8b8-8858fe4121df + - a73df78c-f669-42d6-b8c9-c4fbc57e69fa