From cc154c3e032dfa23f85d190e7a00ee16bf881b1b Mon Sep 17 00:00:00 2001 From: "Md. Sarwar Hossain" Date: Wed, 13 May 2026 12:44:23 +0600 Subject: [PATCH 1/3] feat: update model registry and implement utility scripts for listing and updating benchmarks --- benchmark.py | 3 +- list_models.py | 29 ++ models.json | 442 ++++++++++++++++++- results.json | 1062 ++++++++++++++++++++++++++++++++++++++++------ update_models.py | 58 +++ 5 files changed, 1431 insertions(+), 163 deletions(-) create mode 100644 list_models.py create mode 100644 update_models.py diff --git a/benchmark.py b/benchmark.py index 2c1316b..d79d78f 100644 --- a/benchmark.py +++ b/benchmark.py @@ -1,9 +1,10 @@ import os import json import time -import sys import argparse +# pyrefly: ignore [missing-import] from openai import OpenAI +# pyrefly: ignore [missing-import] from dotenv import load_dotenv load_dotenv() diff --git a/list_models.py b/list_models.py new file mode 100644 index 0000000..26d910b --- /dev/null +++ b/list_models.py @@ -0,0 +1,29 @@ +import os +# pyrefly: ignore [missing-import] +from openai import OpenAI +# pyrefly: ignore [missing-import] +from dotenv import load_dotenv +import json + +load_dotenv() + +API_KEY = os.getenv("NVIDIA_API_KEY") +BASE_URL = "https://integrate.api.nvidia.com/v1" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + +try: + models = client.models.list() + model_list = [] + for model in models.data: + model_list.append({ + "id": model.id, + "label": model.id.split('/')[-1].replace('-', ' ').title() + }) + + # Sort by ID for consistency + model_list.sort(key=lambda x: x["id"]) + + print(json.dumps(model_list, indent=2)) +except Exception as e: + print(f"Error: {e}") diff --git a/models.json b/models.json index 2db8cee..67a8f3f 100644 --- a/models.json +++ b/models.json @@ -1,22 +1,422 @@ [ - { "id": "nvidia/nemotron-3-content-safety", "label": "Nemotron Content Safety" }, - { "id": "z-ai/glm4.7", "label": "GLM 4.7" }, - { "id": "minimaxai/minimax-m2.7", "label": "MiniMax M2.7" }, - { "id": "stepfun-ai/step-3.5-flash", "label": "Step 3.5 Flash" }, - { "id": "moonshotai/kimi-k2-thinking", "label": "Kimi K2 Thinking" }, - { "id": "mistralai/mistral-large-3-675b-instruct-2512", "label": "Mistral Large 3 675B" }, - { "id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", "label": "Nemotron Safety Guard 8B" }, - { "id": "bytedance/seed-oss-36b-instruct", "label": "Seed OSS 36B" }, - { "id": "qwen/qwen3-coder-480b-a35b-instruct", "label": "Qwen3 Coder 480B" }, - { "id": "moonshotai/kimi-k2-instruct", "label": "Kimi K2" }, - { "id": "mistralai/magistral-small-2506", "label": "Magistral Small" }, - { "id": "meta/llama-guard-4-12b", "label": "Llama Guard 4 12B" }, - { "id": "google/gemma-3n-e4b-it", "label": "Gemma 3n E4B" }, - { "id": "google/gemma-3n-e2b-it", "label": "Gemma 3n E2B" }, - { "id": "mistralai/mistral-nemotron", "label": "Mistral Nemotron" }, - { "id": "meta/llama-4-maverick-17b-128e-instruct", "label": "Llama-4 Maverick 17B" }, - { "id": "abacusai/dracarys-llama-3.1-70b-instruct", "label": "Dracarys 70B" }, - { "id": "nvidia/nemotron-mini-4b-instruct", "label": "Nemotron Mini 4B" }, - { "id": "google/gemma-2-2b-it", "label": "Gemma 2 2B" }, - { "id": "upstage/solar-10.7b-instruct", "label": "Solar 10.7B" } -] + { + "id": "01-ai/yi-large", + "label": "Yi Large" + }, + { + "id": "abacusai/dracarys-llama-3.1-70b-instruct", + "label": "Dracarys Llama 3.1 70B Instruct" + }, + { + "id": "adept/fuyu-8b", + "label": "Fuyu 8B" + }, + { + "id": "ai21labs/jamba-1.5-large-instruct", + "label": "Jamba 1.5 Large Instruct" + }, + { + "id": "aisingapore/sea-lion-7b-instruct", + "label": "Sea Lion 7B Instruct" + }, + { + "id": "bigcode/starcoder2-15b", + "label": "Starcoder2 15B" + }, + { + "id": "bytedance/seed-oss-36b-instruct", + "label": "Seed Oss 36B Instruct" + }, + { + "id": "databricks/dbrx-instruct", + "label": "Dbrx Instruct" + }, + { + "id": "deepseek-ai/deepseek-coder-6.7b-instruct", + "label": "Deepseek Coder 6.7B Instruct" + }, + { + "id": "deepseek-ai/deepseek-v4-flash", + "label": "Deepseek V4 Flash" + }, + { + "id": "deepseek-ai/deepseek-v4-pro", + "label": "Deepseek V4 Pro" + }, + { + "id": "google/codegemma-1.1-7b", + "label": "Codegemma 1.1 7B" + }, + { + "id": "google/codegemma-7b", + "label": "Codegemma 7B" + }, + { + "id": "google/deplot", + "label": "Deplot" + }, + { + "id": "google/gemma-2-2b-it", + "label": "Gemma 2 2B Instruct" + }, + { + "id": "google/gemma-2b", + "label": "Gemma 2B" + }, + { + "id": "google/gemma-3-12b-it", + "label": "Gemma 3 12B Instruct" + }, + { + "id": "google/gemma-3-4b-it", + "label": "Gemma 3 4B Instruct" + }, + { + "id": "google/gemma-3n-e2b-it", + "label": "Gemma 3N E2B Instruct" + }, + { + "id": "google/gemma-3n-e4b-it", + "label": "Gemma 3N E4B Instruct" + }, + { + "id": "google/gemma-4-31b-it", + "label": "Gemma 4 31B Instruct" + }, + { + "id": "google/recurrentgemma-2b", + "label": "Recurrentgemma 2B" + }, + { + "id": "ibm/granite-3.0-3b-a800m-instruct", + "label": "Granite 3.0 3B A800M Instruct" + }, + { + "id": "ibm/granite-3.0-8b-instruct", + "label": "Granite 3.0 8B Instruct" + }, + { + "id": "ibm/granite-34b-code-instruct", + "label": "Granite 34B Code Instruct" + }, + { + "id": "ibm/granite-8b-code-instruct", + "label": "Granite 8B Code Instruct" + }, + { + "id": "meta/codellama-70b", + "label": "Codellama 70B" + }, + { + "id": "meta/llama-3.1-70b-instruct", + "label": "Llama 3.1 70B Instruct" + }, + { + "id": "meta/llama-3.1-8b-instruct", + "label": "Llama 3.1 8B Instruct" + }, + { + "id": "meta/llama-3.2-11b-vision-instruct", + "label": "Llama 3.2 11B Vision Instruct" + }, + { + "id": "meta/llama-3.2-1b-instruct", + "label": "Llama 3.2 1B Instruct" + }, + { + "id": "meta/llama-3.2-3b-instruct", + "label": "Llama 3.2 3B Instruct" + }, + { + "id": "meta/llama-3.2-90b-vision-instruct", + "label": "Llama 3.2 90B Vision Instruct" + }, + { + "id": "meta/llama-3.3-70b-instruct", + "label": "Llama 3.3 70B Instruct" + }, + { + "id": "meta/llama-4-maverick-17b-128e-instruct", + "label": "Llama 4 Maverick 17B 128E Instruct" + }, + { + "id": "meta/llama-guard-4-12b", + "label": "Llama Guard 4 12B" + }, + { + "id": "meta/llama2-70b", + "label": "Llama2 70B" + }, + { + "id": "microsoft/kosmos-2", + "label": "Kosmos 2" + }, + { + "id": "microsoft/phi-3-vision-128k-instruct", + "label": "Phi 3 Vision 128K Instruct" + }, + { + "id": "microsoft/phi-3.5-moe-instruct", + "label": "Phi 3.5 Moe Instruct" + }, + { + "id": "microsoft/phi-4-mini-instruct", + "label": "Phi 4 Mini Instruct" + }, + { + "id": "microsoft/phi-4-multimodal-instruct", + "label": "Phi 4 Multimodal Instruct" + }, + { + "id": "minimaxai/minimax-m2.7", + "label": "Minimax M2.7" + }, + { + "id": "mistralai/codestral-22b-instruct-v0.1", + "label": "Codestral 22B Instruct V0.1" + }, + { + "id": "mistralai/ministral-14b-instruct-2512", + "label": "Ministral 14B Instruct 2512" + }, + { + "id": "mistralai/mistral-7b-instruct-v0.3", + "label": "Mistral 7B Instruct V0.3" + }, + { + "id": "mistralai/mistral-large", + "label": "Mistral Large" + }, + { + "id": "mistralai/mistral-large-2-instruct", + "label": "Mistral Large 2 Instruct" + }, + { + "id": "mistralai/mistral-large-3-675b-instruct-2512", + "label": "Mistral Large 3 675B Instruct 2512" + }, + { + "id": "mistralai/mistral-medium-3.5-128b", + "label": "Mistral Medium 3.5 128B" + }, + { + "id": "mistralai/mistral-nemotron", + "label": "Mistral Nemotron" + }, + { + "id": "mistralai/mistral-small-4-119b-2603", + "label": "Mistral Small 4 119B 2603" + }, + { + "id": "mistralai/mixtral-8x22b-instruct-v0.1", + "label": "Mixtral 8X22B Instruct V0.1" + }, + { + "id": "mistralai/mixtral-8x22b-v0.1", + "label": "Mixtral 8X22B V0.1" + }, + { + "id": "mistralai/mixtral-8x7b-instruct-v0.1", + "label": "Mixtral 8X7B Instruct V0.1" + }, + { + "id": "moonshotai/kimi-k2.6", + "label": "Kimi K2.6" + }, + { + "id": "nv-mistralai/mistral-nemo-12b-instruct", + "label": "Mistral Nemo 12B Instruct" + }, + { + "id": "nvidia/cosmos-reason2-8b", + "label": "Cosmos Reason2 8B" + }, + { + "id": "nvidia/gliner-pii", + "label": "Gliner PII" + }, + { + "id": "nvidia/llama-3.1-nemoguard-8b-content-safety", + "label": "Llama 3.1 Nemoguard 8B Content Safety" + }, + { + "id": "nvidia/llama-3.1-nemoguard-8b-topic-control", + "label": "Llama 3.1 Nemoguard 8B Topic Control" + }, + { + "id": "nvidia/llama-3.1-nemotron-51b-instruct", + "label": "Llama 3.1 Nemotron 51B Instruct" + }, + { + "id": "nvidia/llama-3.1-nemotron-70b-instruct", + "label": "Llama 3.1 Nemotron 70B Instruct" + }, + { + "id": "nvidia/llama-3.1-nemotron-nano-8b-v1", + "label": "Llama 3.1 Nemotron Nano 8B V1" + }, + { + "id": "nvidia/llama-3.1-nemotron-nano-vl-8b-v1", + "label": "Llama 3.1 Nemotron Nano VL 8B V1" + }, + { + "id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "label": "Llama 3.1 Nemotron Safety Guard 8B V3" + }, + { + "id": "nvidia/llama-3.1-nemotron-ultra-253b-v1", + "label": "Llama 3.1 Nemotron Ultra 253B V1" + }, + { + "id": "nvidia/llama-3.3-nemotron-super-49b-v1", + "label": "Llama 3.3 Nemotron Super 49B V1" + }, + { + "id": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "label": "Llama 3.3 Nemotron Super 49B V1.5" + }, + { + "id": "nvidia/llama3-chatqa-1.5-70b", + "label": "Llama3 Chatqa 1.5 70B" + }, + { + "id": "nvidia/mistral-nemo-minitron-8b-8k-instruct", + "label": "Mistral Nemo Minitron 8B 8K Instruct" + }, + { + "id": "nvidia/nemotron-3-content-safety", + "label": "Nemotron 3 Content Safety" + }, + { + "id": "nvidia/nemotron-3-nano-30b-a3b", + "label": "Nemotron 3 Nano 30B A3B" + }, + { + "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "label": "Nemotron 3 Nano Omni 30B A3B Reasoning" + }, + { + "id": "nvidia/nemotron-3-super-120b-a12b", + "label": "Nemotron 3 Super 120B A12B" + }, + { + "id": "nvidia/nemotron-4-340b-instruct", + "label": "Nemotron 4 340B Instruct" + }, + { + "id": "nvidia/nemotron-4-340b-reward", + "label": "Nemotron 4 340B Reward" + }, + { + "id": "nvidia/nemotron-content-safety-reasoning-4b", + "label": "Nemotron Content Safety Reasoning 4B" + }, + { + "id": "nvidia/nemotron-mini-4b-instruct", + "label": "Nemotron Mini 4B Instruct" + }, + { + "id": "nvidia/nemotron-nano-12b-v2-vl", + "label": "Nemotron Nano 12B V2 VL" + }, + { + "id": "nvidia/nemotron-nano-3-30b-a3b", + "label": "Nemotron Nano 3 30B A3B" + }, + { + "id": "nvidia/neva-22b", + "label": "Neva 22B" + }, + { + "id": "nvidia/nvidia-nemotron-nano-9b-v2", + "label": "Nvidia Nemotron Nano 9B V2" + }, + { + "id": "nvidia/riva-translate-4b-instruct", + "label": "Riva Translate 4B Instruct" + }, + { + "id": "nvidia/riva-translate-4b-instruct-v1.1", + "label": "Riva Translate 4B Instruct V1.1" + }, + { + "id": "nvidia/vila", + "label": "Vila" + }, + { + "id": "openai/gpt-oss-120b", + "label": "Gpt Oss 120B" + }, + { + "id": "openai/gpt-oss-20b", + "label": "Gpt Oss 20B" + }, + { + "id": "qwen/qwen3-coder-480b-a35b-instruct", + "label": "Qwen3 Coder 480B A35B Instruct" + }, + { + "id": "qwen/qwen3-next-80b-a3b-instruct", + "label": "Qwen3 Next 80B A3B Instruct" + }, + { + "id": "qwen/qwen3-next-80b-a3b-thinking", + "label": "Qwen3 Next 80B A3B Thinking" + }, + { + "id": "qwen/qwen3.5-122b-a10b", + "label": "Qwen3.5 122B A10B" + }, + { + "id": "qwen/qwen3.5-397b-a17b", + "label": "Qwen3.5 397B A17B" + }, + { + "id": "sarvamai/sarvam-m", + "label": "Sarvam M" + }, + { + "id": "stepfun-ai/step-3.5-flash", + "label": "Step 3.5 Flash" + }, + { + "id": "stockmark/stockmark-2-100b-instruct", + "label": "Stockmark 2 100B Instruct" + }, + { + "id": "upstage/solar-10.7b-instruct", + "label": "Solar 10.7B Instruct" + }, + { + "id": "writer/palmyra-creative-122b", + "label": "Palmyra Creative 122B" + }, + { + "id": "writer/palmyra-fin-70b-32k", + "label": "Palmyra Fin 70B 32K" + }, + { + "id": "writer/palmyra-med-70b", + "label": "Palmyra Med 70B" + }, + { + "id": "writer/palmyra-med-70b-32k", + "label": "Palmyra Med 70B 32K" + }, + { + "id": "z-ai/glm-5.1", + "label": "Glm 5.1" + }, + { + "id": "z-ai/glm4.7", + "label": "Glm4.7" + }, + { + "id": "z-ai/glm5", + "label": "Glm5" + }, + { + "id": "zyphra/zamba2-7b-instruct", + "label": "Zamba2 7B Instruct" + } +] \ No newline at end of file diff --git a/results.json b/results.json index 40c0097..80abc1a 100644 --- a/results.json +++ b/results.json @@ -1,262 +1,1042 @@ [ { - "id": "nvidia/nemotron-3-content-safety", - "label": "Nemotron Content Safety", + "id": "01-ai/yi-large", + "label": "Yi Large", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '23bd454d-b225-49a3-8118-582a62fc51b8': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "abacusai/dracarys-llama-3.1-70b-instruct", + "label": "Dracarys Llama 3.1 70B Instruct", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "adept/fuyu-8b", + "label": "Fuyu 8B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'e598bfc1-b058-41af-869d-556d3c7e1b48': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "ai21labs/jamba-1.5-large-instruct", + "label": "Jamba 1.5 Large Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '6497fc2b-7ff8-4019-8946-123dccbfc863': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "aisingapore/sea-lion-7b-instruct", + "label": "Sea Lion 7B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function id '02f84bf4-c1a1-489b-a9de-ac3e8dcdec14' version 'null': Specified function in account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmA", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "bigcode/starcoder2-15b", + "label": "Starcoder2 15B", + "status": "error", + "error": "404 page not found", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "bytedance/seed-oss-36b-instruct", + "label": "Seed Oss 36B Instruct", + "status": "ok", + "total_time_s": 6.143, + "tokens": 8, + "tokens_per_second": 1.3, + "ttft_s": 5.455, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "databricks/dbrx-instruct", + "label": "Dbrx Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '3d6c2ff8-8bfc-4d10-8fd0-b7337288e869': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "deepseek-ai/deepseek-coder-6.7b-instruct", + "label": "Deepseek Coder 6.7B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'e503b15c-62b0-4d69-b532-a88f0bfa2656': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "deepseek-ai/deepseek-v4-flash", + "label": "Deepseek V4 Flash", + "status": "ok", + "total_time_s": 44.446, + "tokens": 2, + "tokens_per_second": 0.04, + "ttft_s": 43.504, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "deepseek-ai/deepseek-v4-pro", + "label": "Deepseek V4 Pro", + "status": "ok", + "total_time_s": 70.831, + "tokens": 2, + "tokens_per_second": 0.03, + "ttft_s": 67.799, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "google/codegemma-1.1-7b", + "label": "Codegemma 1.1 7B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'e2d298c5-204e-4213-b921-9f492cc9011b': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "google/codegemma-7b", + "label": "Codegemma 7B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '7dfc10a8-3cc4-448e-97c1-2213308dc222': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "google/deplot", + "label": "Deplot", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '784a8ca4-ea7d-4c93-bb46-ec027c3fae47': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "google/gemma-2-2b-it", + "label": "Gemma 2 2B Instruct", + "status": "ok", + "total_time_s": 0.443, + "tokens": 8, + "tokens_per_second": 18.05, + "ttft_s": 0.313, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "google/gemma-2b", + "label": "Gemma 2B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '04174188-f742-4069-9e72-d77c2b77d3cb': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "google/gemma-3-12b-it", + "label": "Gemma 3 12B Instruct", + "status": "ok", + "total_time_s": 157.512, + "tokens": 1, + "tokens_per_second": 0.01, + "ttft_s": 157.433, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "google/gemma-3-4b-it", + "label": "Gemma 3 4B Instruct", + "status": "ok", + "total_time_s": 2.074, + "tokens": 1, + "tokens_per_second": 0.48, + "ttft_s": 1.997, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "google/gemma-3n-e2b-it", + "label": "Gemma 3N E2B Instruct", + "status": "ok", + "total_time_s": 0.879, + "tokens": 10, + "tokens_per_second": 11.38, + "ttft_s": 0.356, + "response_preview": "The capital of India is New Delhi. \n" + }, + { + "id": "google/gemma-3n-e4b-it", + "label": "Gemma 3N E4B Instruct", + "status": "ok", + "total_time_s": 0.938, + "tokens": 10, + "tokens_per_second": 10.67, + "ttft_s": 0.353, + "response_preview": "The capital of India is New Delhi. \n" + }, + { + "id": "google/gemma-4-31b-it", + "label": "Gemma 4 31B Instruct", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "google/recurrentgemma-2b", + "label": "Recurrentgemma 2B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '2f495340-a99f-4b4b-89bd-1beb003dd896': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "ibm/granite-3.0-3b-a800m-instruct", + "label": "Granite 3.0 3B A800M Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '67324577-3f91-4aa6-b750-97468262530d': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "ibm/granite-3.0-8b-instruct", + "label": "Granite 3.0 8B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '5a24a4f0-2d59-46b3-ac65-42307f2633d1': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "ibm/granite-34b-code-instruct", + "label": "Granite 34B Code Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '4df48b4f-e3c5-4ade-82c7-c06b65e25d18': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "ibm/granite-8b-code-instruct", + "label": "Granite 8B Code Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'af7b6f03-f615-4c5f-86c6-388bd35cede0': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "meta/codellama-70b", + "label": "Codellama 70B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'f6b06895-d073-4714-8bb2-26c09e9f6597': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "meta/llama-3.1-70b-instruct", + "label": "Llama 3.1 70B Instruct", + "status": "ok", + "total_time_s": 1.226, + "tokens": 8, + "tokens_per_second": 6.53, + "ttft_s": 0.699, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-3.1-8b-instruct", + "label": "Llama 3.1 8B Instruct", + "status": "ok", + "total_time_s": 0.828, + "tokens": 8, + "tokens_per_second": 9.66, + "ttft_s": 0.551, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-3.2-11b-vision-instruct", + "label": "Llama 3.2 11B Vision Instruct", + "status": "ok", + "total_time_s": 5.311, + "tokens": 8, + "tokens_per_second": 1.51, + "ttft_s": 5.006, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-3.2-1b-instruct", + "label": "Llama 3.2 1B Instruct", + "status": "ok", + "total_time_s": 0.777, + "tokens": 8, + "tokens_per_second": 10.3, + "ttft_s": 0.385, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-3.2-3b-instruct", + "label": "Llama 3.2 3B Instruct", + "status": "ok", + "total_time_s": 0.423, + "tokens": 8, + "tokens_per_second": 18.93, + "ttft_s": 0.32, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-3.2-90b-vision-instruct", + "label": "Llama 3.2 90B Vision Instruct", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "meta/llama-3.3-70b-instruct", + "label": "Llama 3.3 70B Instruct", + "status": "ok", + "total_time_s": 5.399, + "tokens": 30, + "tokens_per_second": 5.56, + "ttft_s": 4.854, + "response_preview": "The capital of India is New Delhi, which has been the country's capital since 1911 and is home to numerous historical la" + }, + { + "id": "meta/llama-4-maverick-17b-128e-instruct", + "label": "Llama 4 Maverick 17B 128E Instruct", + "status": "ok", + "total_time_s": 0.667, + "tokens": 8, + "tokens_per_second": 12.0, + "ttft_s": 0.433, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "meta/llama-guard-4-12b", + "label": "Llama Guard 4 12B", + "status": "ok", + "total_time_s": 0.662, + "tokens": 1, + "tokens_per_second": 1.51, + "ttft_s": 0.661, + "response_preview": "safe" + }, + { + "id": "meta/llama2-70b", + "label": "Llama2 70B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '2fddadfb-7e76-4c8a-9b82-f7d3fab94471': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "microsoft/kosmos-2", + "label": "Kosmos 2", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '6018fed7-f227-48dc-99bc-3fd4264d5037': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "microsoft/phi-3-vision-128k-instruct", + "label": "Phi 3 Vision 128K Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '20f2537e-8593-4eb9-ad40-60eee3bbaa55': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "microsoft/phi-3.5-moe-instruct", + "label": "Phi 3.5 Moe Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'e6cab982-62f4-481e-9a7a-3dedb87dbd01': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "microsoft/phi-4-mini-instruct", + "label": "Phi 4 Mini Instruct", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "microsoft/phi-4-multimodal-instruct", + "label": "Phi 4 Multimodal Instruct", + "status": "ok", + "total_time_s": 101.963, + "tokens": 6, + "tokens_per_second": 0.06, + "ttft_s": 101.806, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "minimaxai/minimax-m2.7", + "label": "Minimax M2.7", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "mistralai/codestral-22b-instruct-v0.1", + "label": "Codestral 22B Instruct V0.1", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function id '9a10b012-e6df-46fd-83b2-700dcbc75814' version 'null': Specified function in account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmA", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "mistralai/ministral-14b-instruct-2512", + "label": "Ministral 14B Instruct 2512", + "status": "ok", + "total_time_s": 0.606, + "tokens": 9, + "tokens_per_second": 14.84, + "ttft_s": 0.403, + "response_preview": "The capital of India is **New Delhi**." + }, + { + "id": "mistralai/mistral-7b-instruct-v0.3", + "label": "Mistral 7B Instruct V0.3", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function id 'cd89bd68-13e3-47a9-861e-9a62e6e14b05' version 'null': Specified function in account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmA", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "mistralai/mistral-large", + "label": "Mistral Large", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '767b5b9a-3f9d-4c1d-86e8-fa861988cee7': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "mistralai/mistral-large-2-instruct", + "label": "Mistral Large 2 Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '7fadd4de-e22a-48e4-90e9-f02ef14a74b9': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "mistralai/mistral-large-3-675b-instruct-2512", + "label": "Mistral Large 3 675B Instruct 2512", "status": "ok", - "total_time_s": 1.084, - "tokens": 4, - "tokens_per_second": 3.69, - "ttft_s": 1.045, - "response_preview": "User Safety: safe" + "total_time_s": 102.639, + "tokens": 9, + "tokens_per_second": 0.09, + "ttft_s": 100.946, + "response_preview": "The capital of India is **New Delhi**." }, { - "id": "z-ai/glm4.7", - "label": "GLM 4.7", + "id": "mistralai/mistral-medium-3.5-128b", + "label": "Mistral Medium 3.5 128B", "status": "ok", - "total_time_s": 6.919, - "tokens": 3, - "tokens_per_second": 0.43, - "ttft_s": 6.641, - "response_preview": "The capital of India is New Delhi." + "total_time_s": 0.592, + "tokens": 9, + "tokens_per_second": 15.2, + "ttft_s": 0.367, + "response_preview": "The capital of India is **New Delhi**." }, { - "id": "minimaxai/minimax-m2.7", - "label": "MiniMax M2.7", + "id": "mistralai/mistral-nemotron", + "label": "Mistral Nemotron", "status": "ok", - "total_time_s": 10.763, - "tokens": 11, - "tokens_per_second": 1.02, - "ttft_s": 10.595, - "response_preview": "\n\nThe capital of India is New\u202fDelhi." + "total_time_s": 1.428, + "tokens": 8, + "tokens_per_second": 5.6, + "ttft_s": 0.526, + "response_preview": "The capital of India is New Delhi." }, { - "id": "stepfun-ai/step-3.5-flash", - "label": "Step 3.5 Flash", + "id": "mistralai/mistral-small-4-119b-2603", + "label": "Mistral Small 4 119B 2603", "status": "ok", - "total_time_s": 2.569, - "tokens": 2, - "tokens_per_second": 0.78, - "ttft_s": 2.486, + "total_time_s": 1.583, + "tokens": 8, + "tokens_per_second": 5.05, + "ttft_s": 0.708, "response_preview": "The capital of India is New Delhi." }, { - "id": "mistralai/devstral-2-123b-instruct-2512", - "label": "Devstral 2 123B", + "id": "mistralai/mixtral-8x22b-instruct-v0.1", + "label": "Mixtral 8X22B Instruct V0.1", + "status": "ok", + "total_time_s": 4.939, + "tokens": 8, + "tokens_per_second": 1.62, + "ttft_s": 0.426, + "response_preview": " The capital of India is New Delhi." + }, + { + "id": "mistralai/mixtral-8x22b-v0.1", + "label": "Mixtral 8X22B V0.1", "status": "error", - "error": "Error code: 400 - {'status': 400, 'title': 'Bad Request', 'detail': \"Function id '7fe236cd-dab4-40d4-a139-b28d6673ffd3': DEGRADED function cannot be invoked\"}", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function id '39655fc1-9ebc-4b24-963e-6915ea6680de' version 'null': Specified function in account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmA", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "moonshotai/kimi-k2-thinking", - "label": "Kimi K2 Thinking", + "id": "mistralai/mixtral-8x7b-instruct-v0.1", + "label": "Mixtral 8X7B Instruct V0.1", "status": "ok", - "total_time_s": 13.172, + "total_time_s": 8.322, + "tokens": 63, + "tokens_per_second": 7.57, + "ttft_s": 0.432, + "response_preview": " The capital of India is New Delhi, which serves as the central hub for administrative purposes.\n\nWhile I strive to prov" + }, + { + "id": "moonshotai/kimi-k2.6", + "label": "Kimi K2.6", + "status": "ok", + "total_time_s": 88.583, "tokens": 2, - "tokens_per_second": 0.15, - "ttft_s": 12.973, - "response_preview": " New Delhi is the capital of India." + "tokens_per_second": 0.02, + "ttft_s": 88.366, + "response_preview": "The capital of India is New Delhi." }, { - "id": "mistralai/mistral-large-3-675b-instruct-2512", - "label": "Mistral Large 3 675B", + "id": "nv-mistralai/mistral-nemo-12b-instruct", + "label": "Mistral Nemo 12B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'f8c05193-d2e2-4f0f-bb4d-7ad70070002b': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/cosmos-reason2-8b", + "label": "Cosmos Reason2 8B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'e199b43b-6c62-4a63-9379-f60e1a953236': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/gliner-pii", + "label": "Gliner PII", "status": "ok", - "total_time_s": 2.49, - "tokens": 9, - "tokens_per_second": 3.61, - "ttft_s": 1.16, - "response_preview": "The capital of India is **New Delhi**." + "total_time_s": 0.335, + "tokens": 0, + "tokens_per_second": 0.0, + "ttft_s": null, + "response_preview": "" }, { - "id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", - "label": "Nemotron Safety Guard 8B", + "id": "nvidia/llama-3.1-nemoguard-8b-content-safety", + "label": "Llama 3.1 Nemoguard 8B Content Safety", "status": "ok", - "total_time_s": 0.575, + "total_time_s": 0.713, "tokens": 8, - "tokens_per_second": 13.92, - "ttft_s": 0.418, + "tokens_per_second": 11.22, + "ttft_s": 0.709, "response_preview": "{\"User Safety\": \"safe\"} " }, { - "id": "bytedance/seed-oss-36b-instruct", - "label": "Seed OSS 36B", + "id": "nvidia/llama-3.1-nemoguard-8b-topic-control", + "label": "Llama 3.1 Nemoguard 8B Topic Control", + "status": "ok", + "total_time_s": 0.346, + "tokens": 3, + "tokens_per_second": 8.67, + "ttft_s": 0.311, + "response_preview": "on-topic " + }, + { + "id": "nvidia/llama-3.1-nemotron-51b-instruct", + "label": "Llama 3.1 Nemotron 51B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '5beba52c-65a9-4f46-8cd9-656689a1b205': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/llama-3.1-nemotron-70b-instruct", + "label": "Llama 3.1 Nemotron 70B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '9b96341b-9791-4db9-a00d-4e43aa192a39': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/llama-3.1-nemotron-nano-8b-v1", + "label": "Llama 3.1 Nemotron Nano 8B V1", "status": "ok", - "total_time_s": 74.301, + "total_time_s": 8.896, "tokens": 8, - "tokens_per_second": 0.11, - "ttft_s": 74.065, - "response_preview": "The capital of India is New Delhi." + "tokens_per_second": 0.9, + "ttft_s": 1.531, + "response_preview": " The capital of India is New Delhi." }, { - "id": "qwen/qwen3-coder-480b-a35b-instruct", - "label": "Qwen3 Coder 480B", + "id": "nvidia/llama-3.1-nemotron-nano-vl-8b-v1", + "label": "Llama 3.1 Nemotron Nano VL 8B V1", "status": "ok", - "total_time_s": 148.918, - "tokens": 4, - "tokens_per_second": 0.03, - "ttft_s": 148.41, - "response_preview": "The capital of India is New Delhi." + "total_time_s": 0.531, + "tokens": 8, + "tokens_per_second": 15.07, + "ttft_s": 0.335, + "response_preview": "New Delhi is the capital of India." + }, + { + "id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", + "label": "Llama 3.1 Nemotron Safety Guard 8B V3", + "status": "ok", + "total_time_s": 0.496, + "tokens": 8, + "tokens_per_second": 16.14, + "ttft_s": 0.346, + "response_preview": "{\"User Safety\": \"safe\"} " }, { - "id": "nvidia/llama-3.2-nemoretriever-300m-embed-v1", - "label": "NemoRetriever 300M", + "id": "nvidia/llama-3.1-nemotron-ultra-253b-v1", + "label": "Llama 3.1 Nemotron Ultra 253B V1", "status": "error", - "error": "404 page not found", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '84bf12ff-edbd-4435-baea-0fa6a7453d2e': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "moonshotai/kimi-k2-instruct", - "label": "Kimi K2", + "id": "nvidia/llama-3.3-nemotron-super-49b-v1", + "label": "Llama 3.3 Nemotron Super 49B V1", "status": "ok", - "total_time_s": 0.872, + "total_time_s": 0.846, "tokens": 2, - "tokens_per_second": 2.29, - "ttft_s": 0.66, + "tokens_per_second": 2.36, + "ttft_s": 0.437, "response_preview": "The capital of India is New Delhi." }, { - "id": "mistralai/magistral-small-2506", - "label": "Magistral Small", + "id": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "label": "Llama 3.3 Nemotron Super 49B V1.5", "status": "ok", - "total_time_s": 11.603, + "total_time_s": 9.406, + "tokens": 2, + "tokens_per_second": 0.21, + "ttft_s": 9.304, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "nvidia/llama3-chatqa-1.5-70b", + "label": "Llama3 Chatqa 1.5 70B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '46594287-38b9-481c-a37f-baa02f2d3ba1': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, "tokens": 0, - "tokens_per_second": 0.0, - "ttft_s": null, - "response_preview": "" + "tokens_per_second": 0, + "ttft_s": null }, { - "id": "meta/llama-guard-4-12b", - "label": "Llama Guard 4 12B", + "id": "nvidia/mistral-nemo-minitron-8b-8k-instruct", + "label": "Mistral Nemo Minitron 8B 8K Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '5aa06dd2-0a02-4a5d-be4c-bf88e956965d': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/nemotron-3-content-safety", + "label": "Nemotron 3 Content Safety", "status": "ok", - "total_time_s": 0.821, - "tokens": 1, - "tokens_per_second": 1.22, - "ttft_s": 0.82, - "response_preview": "safe" + "total_time_s": 0.684, + "tokens": 4, + "tokens_per_second": 5.85, + "ttft_s": 0.682, + "response_preview": "User Safety: safe" }, { - "id": "google/gemma-3n-e4b-it", - "label": "Gemma 3n E4B", + "id": "nvidia/nemotron-3-nano-30b-a3b", + "label": "Nemotron 3 Nano 30B A3B", "status": "ok", - "total_time_s": 0.975, - "tokens": 10, - "tokens_per_second": 10.26, - "ttft_s": 0.448, - "response_preview": "The capital of India is New Delhi. \n" + "total_time_s": 0.589, + "tokens": 2, + "tokens_per_second": 3.39, + "ttft_s": 0.547, + "response_preview": "The capital of India is New\u202fDelhi." }, { - "id": "google/gemma-3n-e2b-it", - "label": "Gemma 3n E2B", + "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "label": "Nemotron 3 Nano Omni 30B A3B Reasoning", "status": "ok", - "total_time_s": 0.898, - "tokens": 10, - "tokens_per_second": 11.13, - "ttft_s": 0.442, - "response_preview": "The capital of India is New Delhi. \n" + "total_time_s": 0.571, + "tokens": 2, + "tokens_per_second": 3.51, + "ttft_s": 0.543, + "response_preview": "The capital of India is New\u202fDelhi." }, { - "id": "mistralai/mistral-nemotron", - "label": "Mistral Nemotron", + "id": "nvidia/nemotron-3-super-120b-a12b", + "label": "Nemotron 3 Super 120B A12B", "status": "ok", - "total_time_s": 0.545, - "tokens": 9, - "tokens_per_second": 16.52, - "ttft_s": 0.375, - "response_preview": "The capital of India is **New Delhi**." + "total_time_s": 0.968, + "tokens": 1, + "tokens_per_second": 1.03, + "ttft_s": 0.968, + "response_preview": "The capital of India is New\u202fDelhi." }, { - "id": "mistralai/mistral-medium-3-instruct", - "label": "Mistral Medium 3", + "id": "nvidia/nemotron-4-340b-instruct", + "label": "Nemotron 4 340B Instruct", "status": "error", - "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function id 'e48444ed-3120-443d-b57c-8206fac3b980' version 'null': Specified function in account 'tLOJ1oeRJ3R6E8uzUMVPdVdP0r-n5BNC0LG", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'b0fcd392-e905-4ab4-8eb9-aeae95c30b37': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "meta/llama-4-maverick-17b-128e-instruct", - "label": "Llama-4 Maverick 17B", + "id": "nvidia/nemotron-4-340b-reward", + "label": "Nemotron 4 340B Reward", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'c53ee0e9-bad9-4e09-b365-52c9d6b71254': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "nvidia/nemotron-content-safety-reasoning-4b", + "label": "Nemotron Content Safety Reasoning 4B", + "status": "ok", + "total_time_s": 0.424, + "tokens": 8, + "tokens_per_second": 18.88, + "ttft_s": 0.309, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "nvidia/nemotron-mini-4b-instruct", + "label": "Nemotron Mini 4B Instruct", "status": "ok", - "total_time_s": 4.228, + "total_time_s": 0.406, "tokens": 8, - "tokens_per_second": 1.89, - "ttft_s": 4.116, + "tokens_per_second": 19.71, + "ttft_s": 0.299, "response_preview": "The capital of India is New Delhi." }, { - "id": "google/gemma-3-27b-it", - "label": "Gemma 3 27B", + "id": "nvidia/nemotron-nano-12b-v2-vl", + "label": "Nemotron Nano 12B V2 VL", + "status": "ok", + "total_time_s": 4.095, + "tokens": 8, + "tokens_per_second": 1.95, + "ttft_s": 3.899, + "response_preview": "The capital of India is New Delhi.\n" + }, + { + "id": "nvidia/nemotron-nano-3-30b-a3b", + "label": "Nemotron Nano 3 30B A3B", "status": "error", - "error": "Error code: 400 - {'status': 400, 'title': 'Bad Request', 'detail': \"Function id 'b86482fe-8e57-4f0a-8efd-7a45a985377e': DEGRADED function cannot be invoked\"}", + "error": "Error code: 404 - {'error': {'message': 'The model `nvidia/nemotron-nano-3-30b-a3b` does not exist.', 'type': 'NotFoundError', 'param': 'model', 'code': 404}}", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "microsoft/phi-4-multimodal-instruct", - "label": "Phi-4 Multimodal", + "id": "nvidia/neva-22b", + "label": "Neva 22B", "status": "error", - "error": "Error code: 400 - {'status': 400, 'title': 'Bad Request', 'detail': \"Function id '3afd112d-ad66-4d29-81b6-e7d6f8225e07': DEGRADED function cannot be invoked\"}", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'bc205f8e-1740-40df-8d32-c4321763498a': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "nvidia/usdcode", - "label": "NVIDIA USDCode", + "id": "nvidia/nvidia-nemotron-nano-9b-v2", + "label": "Nvidia Nemotron Nano 9B V2", + "status": "ok", + "total_time_s": 4.932, + "tokens": 2, + "tokens_per_second": 0.41, + "ttft_s": 4.851, + "response_preview": "\n\nThe capital of India is New Delhi.\n" + }, + { + "id": "nvidia/riva-translate-4b-instruct", + "label": "Riva Translate 4B Instruct", "status": "error", - "error": "404 page not found", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'f35337fa-b4dd-4996-bcba-5476ee01171d': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", "total_time_s": null, "tokens": 0, "tokens_per_second": 0, "ttft_s": null }, { - "id": "abacusai/dracarys-llama-3.1-70b-instruct", - "label": "Dracarys 70B", + "id": "nvidia/riva-translate-4b-instruct-v1.1", + "label": "Riva Translate 4B Instruct V1.1", "status": "ok", - "total_time_s": 179.489, - "tokens": 8, - "tokens_per_second": 0.04, - "ttft_s": 173.267, + "total_time_s": 0.437, + "tokens": 13, + "tokens_per_second": 29.72, + "ttft_s": 0.288, + "response_preview": "What is the capital of India? Please answer in one sentence." + }, + { + "id": "nvidia/vila", + "label": "Vila", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '1c8df143-2303-419b-8b28-b4dd82cfe113': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "openai/gpt-oss-120b", + "label": "Gpt Oss 120B", + "status": "ok", + "total_time_s": 1.658, + "tokens": 9, + "tokens_per_second": 5.43, + "ttft_s": 1.6, + "response_preview": "The capital of India is New\u202fDelhi." + }, + { + "id": "openai/gpt-oss-20b", + "label": "Gpt Oss 20B", + "status": "ok", + "total_time_s": 1.058, + "tokens": 4, + "tokens_per_second": 3.78, + "ttft_s": 1.007, "response_preview": "The capital of India is New Delhi." }, { - "id": "nvidia/nemotron-mini-4b-instruct", - "label": "Nemotron Mini 4B", + "id": "qwen/qwen3-coder-480b-a35b-instruct", + "label": "Qwen3 Coder 480B A35B Instruct", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "qwen/qwen3-next-80b-a3b-instruct", + "label": "Qwen3 Next 80B A3B Instruct", "status": "ok", - "total_time_s": 0.531, - "tokens": 8, - "tokens_per_second": 15.07, - "ttft_s": 0.419, + "total_time_s": 4.68, + "tokens": 2, + "tokens_per_second": 0.43, + "ttft_s": 4.569, "response_preview": "The capital of India is New Delhi." }, { - "id": "google/gemma-2-2b-it", - "label": "Gemma 2 2B", + "id": "qwen/qwen3-next-80b-a3b-thinking", + "label": "Qwen3 Next 80B A3B Thinking", + "status": "ok", + "total_time_s": 2.389, + "tokens": 2, + "tokens_per_second": 0.84, + "ttft_s": 2.295, + "response_preview": "\n\nThe capital of India is New Delhi." + }, + { + "id": "qwen/qwen3.5-122b-a10b", + "label": "Qwen3.5 122B A10B", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "qwen/qwen3.5-397b-a17b", + "label": "Qwen3.5 397B A17B", + "status": "ok", + "total_time_s": 134.576, + "tokens": 2, + "tokens_per_second": 0.01, + "ttft_s": 134.458, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "sarvamai/sarvam-m", + "label": "Sarvam M", + "status": "ok", + "total_time_s": 4.029, + "tokens": 119, + "tokens_per_second": 29.53, + "ttft_s": 0.336, + "response_preview": "Okay, the user is asking for the capital of India. Let me make sure I have the correct information. I remember that Indi" + }, + { + "id": "stepfun-ai/step-3.5-flash", + "label": "Step 3.5 Flash", + "status": "ok", + "total_time_s": 4.738, + "tokens": 2, + "tokens_per_second": 0.42, + "ttft_s": 4.706, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "stockmark/stockmark-2-100b-instruct", + "label": "Stockmark 2 100B Instruct", "status": "ok", - "total_time_s": 0.509, + "total_time_s": 0.67, "tokens": 8, - "tokens_per_second": 15.71, - "ttft_s": 0.39, + "tokens_per_second": 11.94, + "ttft_s": 0.369, "response_preview": "The capital of India is New Delhi." }, { "id": "upstage/solar-10.7b-instruct", - "label": "Solar 10.7B", + "label": "Solar 10.7B Instruct", "status": "ok", - "total_time_s": 0.619, + "total_time_s": 0.58, "tokens": 8, - "tokens_per_second": 12.92, - "ttft_s": 0.401, + "tokens_per_second": 13.8, + "ttft_s": 0.366, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "writer/palmyra-creative-122b", + "label": "Palmyra Creative 122B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '00bdd0a7-e38f-4423-9007-c4d8730a3f78': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "writer/palmyra-fin-70b-32k", + "label": "Palmyra Fin 70B 32K", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '316490c6-f1ed-41f9-9da8-3fa9e885653b': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "writer/palmyra-med-70b", + "label": "Palmyra Med 70B", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'aab71274-5281-4941-b0b8-20f339d1fc7e': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "writer/palmyra-med-70b-32k", + "label": "Palmyra Med 70B 32K", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function 'd6faa974-3591-49a4-963d-97221d074b2e': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null + }, + { + "id": "z-ai/glm-5.1", + "label": "Glm 5.1", + "status": "ok", + "total_time_s": 6.025, + "tokens": 2, + "tokens_per_second": 0.33, + "ttft_s": 5.887, + "response_preview": "The capital of India is New Delhi." + }, + { + "id": "z-ai/glm4.7", + "label": "Glm4.7", + "status": "ok", + "total_time_s": 79.479, + "tokens": 3, + "tokens_per_second": 0.04, + "ttft_s": 79.125, "response_preview": "The capital of India is New Delhi." + }, + { + "id": "z-ai/glm5", + "label": "Glm5", + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null } ] \ No newline at end of file diff --git a/update_models.py b/update_models.py new file mode 100644 index 0000000..8059ef2 --- /dev/null +++ b/update_models.py @@ -0,0 +1,58 @@ +import os +# pyrefly: ignore [missing-import] +from openai import OpenAI +# pyrefly: ignore [missing-import] +from dotenv import load_dotenv +import json +import re + +load_dotenv() + +API_KEY = os.getenv("NVIDIA_API_KEY") +BASE_URL = "https://integrate.api.nvidia.com/v1" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + +def format_label(model_id): + name = model_id.split('/')[-1] + name = re.sub(r'[-_]', ' ', name) + name = name.title() + name = name.replace('It', 'Instruct') + name = name.replace('Vl', 'VL') + name = name.replace('Nv ', 'NV ') + name = name.replace('Pii', 'PII') + return name + +try: + models = client.models.list() + seen_ids = set() + model_list = [] + + # Common non-chat model patterns + exclude_patterns = [ + 'embed', 'search', 'rank', 'parse', 'clip', 'detector', + 'calibration', 'bge', 'cosign', 'similarity', 'mnet' + ] + + for model in models.data: + if model.id in seen_ids: + continue + + low_id = model.id.lower() + if any(x in low_id for x in exclude_patterns): + continue + + seen_ids.add(model.id) + model_list.append({ + "id": model.id, + "label": format_label(model.id) + }) + + model_list.sort(key=lambda x: x["id"]) + + with open("models.json", "w") as f: + json.dump(model_list, f, indent=2) + + print(f"Successfully updated models.json with {len(model_list)} unique chat-style models.") +except Exception as e: + print(f"Error: {e}") From 53172698ebc66b930a5b32875205ae3062458535 Mon Sep 17 00:00:00 2001 From: "Md. Sarwar Hossain" Date: Wed, 13 May 2026 12:51:31 +0600 Subject: [PATCH 2/3] chore: add error status entry for zyphra/zamba2-7b-instruct in speed test results --- results.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/results.json b/results.json index 80abc1a..4580b67 100644 --- a/results.json +++ b/results.json @@ -1038,5 +1038,15 @@ "tokens": 0, "tokens_per_second": 0, "ttft_s": null + }, + { + "id": "zyphra/zamba2-7b-instruct", + "label": "Zamba2 7B Instruct", + "status": "error", + "error": "Error code: 404 - {'status': 404, 'title': 'Not Found', 'detail': \"Function '8378ffb2-51b0-4140-9684-dda1889373e6': Not found for account 'DCDUd-TlCelp0PbWonyKNOfK8Q2XTq1QbmAaQTsA4KM'\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null } ] \ No newline at end of file From 262d36a1bc382f97232b3875c485da48414a71f6 Mon Sep 17 00:00:00 2001 From: "Md. Sarwar Hossain" Date: Wed, 13 May 2026 15:34:55 +0600 Subject: [PATCH 3/3] chore: update model speed test results in results.json --- results.json | 400 +++++++++++++++++++++++++-------------------------- 1 file changed, 200 insertions(+), 200 deletions(-) diff --git a/results.json b/results.json index 4580b67..38fe82d 100644 --- a/results.json +++ b/results.json @@ -12,12 +12,12 @@ { "id": "abacusai/dracarys-llama-3.1-70b-instruct", "label": "Dracarys Llama 3.1 70B Instruct", - "status": "error", - "error": "Request timed out.", - "total_time_s": null, - "tokens": 0, - "tokens_per_second": 0, - "ttft_s": null + "status": "ok", + "total_time_s": 149.261, + "tokens": 8, + "tokens_per_second": 0.05, + "ttft_s": 140.625, + "response_preview": "The capital of India is New Delhi." }, { "id": "adept/fuyu-8b", @@ -63,10 +63,10 @@ "id": "bytedance/seed-oss-36b-instruct", "label": "Seed Oss 36B Instruct", "status": "ok", - "total_time_s": 6.143, + "total_time_s": 8.892, "tokens": 8, - "tokens_per_second": 1.3, - "ttft_s": 5.455, + "tokens_per_second": 0.9, + "ttft_s": 8.438, "response_preview": "The capital of India is New Delhi." }, { @@ -93,21 +93,21 @@ "id": "deepseek-ai/deepseek-v4-flash", "label": "Deepseek V4 Flash", "status": "ok", - "total_time_s": 44.446, + "total_time_s": 74.697, "tokens": 2, - "tokens_per_second": 0.04, - "ttft_s": 43.504, + "tokens_per_second": 0.03, + "ttft_s": 74.344, "response_preview": "The capital of India is New Delhi." }, { "id": "deepseek-ai/deepseek-v4-pro", "label": "Deepseek V4 Pro", - "status": "ok", - "total_time_s": 70.831, - "tokens": 2, - "tokens_per_second": 0.03, - "ttft_s": 67.799, - "response_preview": "The capital of India is New Delhi." + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null }, { "id": "google/codegemma-1.1-7b", @@ -143,10 +143,10 @@ "id": "google/gemma-2-2b-it", "label": "Gemma 2 2B Instruct", "status": "ok", - "total_time_s": 0.443, + "total_time_s": 1.075, "tokens": 8, - "tokens_per_second": 18.05, - "ttft_s": 0.313, + "tokens_per_second": 7.44, + "ttft_s": 0.312, "response_preview": "The capital of India is New Delhi." }, { @@ -163,40 +163,40 @@ "id": "google/gemma-3-12b-it", "label": "Gemma 3 12B Instruct", "status": "ok", - "total_time_s": 157.512, + "total_time_s": 12.254, "tokens": 1, - "tokens_per_second": 0.01, - "ttft_s": 157.433, + "tokens_per_second": 0.08, + "ttft_s": 12.174, "response_preview": "The capital of India is New Delhi." }, { "id": "google/gemma-3-4b-it", "label": "Gemma 3 4B Instruct", "status": "ok", - "total_time_s": 2.074, + "total_time_s": 6.33, "tokens": 1, - "tokens_per_second": 0.48, - "ttft_s": 1.997, + "tokens_per_second": 0.16, + "ttft_s": 6.278, "response_preview": "The capital of India is New Delhi." }, { "id": "google/gemma-3n-e2b-it", "label": "Gemma 3N E2B Instruct", "status": "ok", - "total_time_s": 0.879, + "total_time_s": 0.907, "tokens": 10, - "tokens_per_second": 11.38, - "ttft_s": 0.356, + "tokens_per_second": 11.03, + "ttft_s": 0.364, "response_preview": "The capital of India is New Delhi. \n" }, { "id": "google/gemma-3n-e4b-it", "label": "Gemma 3N E4B Instruct", "status": "ok", - "total_time_s": 0.938, + "total_time_s": 0.949, "tokens": 10, - "tokens_per_second": 10.67, - "ttft_s": 0.353, + "tokens_per_second": 10.53, + "ttft_s": 0.396, "response_preview": "The capital of India is New Delhi. \n" }, { @@ -273,90 +273,90 @@ "id": "meta/llama-3.1-70b-instruct", "label": "Llama 3.1 70B Instruct", "status": "ok", - "total_time_s": 1.226, + "total_time_s": 5.924, "tokens": 8, - "tokens_per_second": 6.53, - "ttft_s": 0.699, + "tokens_per_second": 1.35, + "ttft_s": 5.486, "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-3.1-8b-instruct", "label": "Llama 3.1 8B Instruct", "status": "ok", - "total_time_s": 0.828, + "total_time_s": 12.13, "tokens": 8, - "tokens_per_second": 9.66, - "ttft_s": 0.551, + "tokens_per_second": 0.66, + "ttft_s": 11.953, "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-3.2-11b-vision-instruct", "label": "Llama 3.2 11B Vision Instruct", "status": "ok", - "total_time_s": 5.311, + "total_time_s": 0.437, "tokens": 8, - "tokens_per_second": 1.51, - "ttft_s": 5.006, + "tokens_per_second": 18.3, + "ttft_s": 0.317, "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-3.2-1b-instruct", "label": "Llama 3.2 1B Instruct", "status": "ok", - "total_time_s": 0.777, + "total_time_s": 0.421, "tokens": 8, - "tokens_per_second": 10.3, - "ttft_s": 0.385, + "tokens_per_second": 19.02, + "ttft_s": 0.304, "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-3.2-3b-instruct", "label": "Llama 3.2 3B Instruct", "status": "ok", - "total_time_s": 0.423, + "total_time_s": 0.413, "tokens": 8, - "tokens_per_second": 18.93, - "ttft_s": 0.32, - "response_preview": "The capital of India is New Delhi." + "tokens_per_second": 19.35, + "ttft_s": 0.303, + "response_preview": "New Delhi is the capital of India." }, { "id": "meta/llama-3.2-90b-vision-instruct", "label": "Llama 3.2 90B Vision Instruct", - "status": "error", - "error": "Request timed out.", - "total_time_s": null, - "tokens": 0, - "tokens_per_second": 0, - "ttft_s": null + "status": "ok", + "total_time_s": 0.781, + "tokens": 8, + "tokens_per_second": 10.24, + "ttft_s": 0.447, + "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-3.3-70b-instruct", "label": "Llama 3.3 70B Instruct", "status": "ok", - "total_time_s": 5.399, - "tokens": 30, - "tokens_per_second": 5.56, - "ttft_s": 4.854, - "response_preview": "The capital of India is New Delhi, which has been the country's capital since 1911 and is home to numerous historical la" + "total_time_s": 0.828, + "tokens": 19, + "tokens_per_second": 22.95, + "ttft_s": 0.328, + "response_preview": "The capital of India is New Delhi, which serves as the country's administrative and political hub." }, { "id": "meta/llama-4-maverick-17b-128e-instruct", "label": "Llama 4 Maverick 17B 128E Instruct", "status": "ok", - "total_time_s": 0.667, - "tokens": 8, - "tokens_per_second": 12.0, - "ttft_s": 0.433, + "total_time_s": 5.629, + "tokens": 6, + "tokens_per_second": 1.07, + "ttft_s": 5.409, "response_preview": "The capital of India is New Delhi." }, { "id": "meta/llama-guard-4-12b", "label": "Llama Guard 4 12B", "status": "ok", - "total_time_s": 0.662, + "total_time_s": 0.696, "tokens": 1, - "tokens_per_second": 1.51, - "ttft_s": 0.661, + "tokens_per_second": 1.44, + "ttft_s": 0.687, "response_preview": "safe" }, { @@ -412,12 +412,12 @@ { "id": "microsoft/phi-4-multimodal-instruct", "label": "Phi 4 Multimodal Instruct", - "status": "ok", - "total_time_s": 101.963, - "tokens": 6, - "tokens_per_second": 0.06, - "ttft_s": 101.806, - "response_preview": "The capital of India is New Delhi." + "status": "error", + "error": "Error code: 400 - {'status': 400, 'title': 'Bad Request', 'detail': \"Function id '3afd112d-ad66-4d29-81b6-e7d6f8225e07': DEGRADED function cannot be invoked\"}", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null }, { "id": "minimaxai/minimax-m2.7", @@ -443,10 +443,10 @@ "id": "mistralai/ministral-14b-instruct-2512", "label": "Ministral 14B Instruct 2512", "status": "ok", - "total_time_s": 0.606, + "total_time_s": 0.531, "tokens": 9, - "tokens_per_second": 14.84, - "ttft_s": 0.403, + "tokens_per_second": 16.93, + "ttft_s": 0.373, "response_preview": "The capital of India is **New Delhi**." }, { @@ -483,50 +483,50 @@ "id": "mistralai/mistral-large-3-675b-instruct-2512", "label": "Mistral Large 3 675B Instruct 2512", "status": "ok", - "total_time_s": 102.639, + "total_time_s": 25.599, "tokens": 9, - "tokens_per_second": 0.09, - "ttft_s": 100.946, + "tokens_per_second": 0.35, + "ttft_s": 24.814, "response_preview": "The capital of India is **New Delhi**." }, { "id": "mistralai/mistral-medium-3.5-128b", "label": "Mistral Medium 3.5 128B", "status": "ok", - "total_time_s": 0.592, + "total_time_s": 10.915, "tokens": 9, - "tokens_per_second": 15.2, - "ttft_s": 0.367, + "tokens_per_second": 0.82, + "ttft_s": 10.692, "response_preview": "The capital of India is **New Delhi**." }, { "id": "mistralai/mistral-nemotron", "label": "Mistral Nemotron", - "status": "ok", - "total_time_s": 1.428, - "tokens": 8, - "tokens_per_second": 5.6, - "ttft_s": 0.526, - "response_preview": "The capital of India is New Delhi." + "status": "error", + "error": "The read operation timed out", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null }, { "id": "mistralai/mistral-small-4-119b-2603", "label": "Mistral Small 4 119B 2603", "status": "ok", - "total_time_s": 1.583, - "tokens": 8, - "tokens_per_second": 5.05, - "ttft_s": 0.708, - "response_preview": "The capital of India is New Delhi." + "total_time_s": 0.802, + "tokens": 9, + "tokens_per_second": 11.22, + "ttft_s": 0.523, + "response_preview": "The capital of India is **New Delhi**." }, { "id": "mistralai/mixtral-8x22b-instruct-v0.1", "label": "Mixtral 8X22B Instruct V0.1", "status": "ok", - "total_time_s": 4.939, + "total_time_s": 0.415, "tokens": 8, - "tokens_per_second": 1.62, - "ttft_s": 0.426, + "tokens_per_second": 19.29, + "ttft_s": 0.291, "response_preview": " The capital of India is New Delhi." }, { @@ -543,20 +543,20 @@ "id": "mistralai/mixtral-8x7b-instruct-v0.1", "label": "Mixtral 8X7B Instruct V0.1", "status": "ok", - "total_time_s": 8.322, - "tokens": 63, - "tokens_per_second": 7.57, - "ttft_s": 0.432, - "response_preview": " The capital of India is New Delhi, which serves as the central hub for administrative purposes.\n\nWhile I strive to prov" + "total_time_s": 1.799, + "tokens": 36, + "tokens_per_second": 20.01, + "ttft_s": 0.442, + "response_preview": " The capital of India is New Delhi, which serves as the administrative hub of the country, while there are also state ca" }, { "id": "moonshotai/kimi-k2.6", "label": "Kimi K2.6", "status": "ok", - "total_time_s": 88.583, + "total_time_s": 35.462, "tokens": 2, - "tokens_per_second": 0.02, - "ttft_s": 88.366, + "tokens_per_second": 0.06, + "ttft_s": 35.201, "response_preview": "The capital of India is New Delhi." }, { @@ -583,7 +583,7 @@ "id": "nvidia/gliner-pii", "label": "Gliner PII", "status": "ok", - "total_time_s": 0.335, + "total_time_s": 0.352, "tokens": 0, "tokens_per_second": 0.0, "ttft_s": null, @@ -593,20 +593,20 @@ "id": "nvidia/llama-3.1-nemoguard-8b-content-safety", "label": "Llama 3.1 Nemoguard 8B Content Safety", "status": "ok", - "total_time_s": 0.713, + "total_time_s": 5.037, "tokens": 8, - "tokens_per_second": 11.22, - "ttft_s": 0.709, + "tokens_per_second": 1.59, + "ttft_s": 5.014, "response_preview": "{\"User Safety\": \"safe\"} " }, { "id": "nvidia/llama-3.1-nemoguard-8b-topic-control", "label": "Llama 3.1 Nemoguard 8B Topic Control", "status": "ok", - "total_time_s": 0.346, + "total_time_s": 0.368, "tokens": 3, - "tokens_per_second": 8.67, - "ttft_s": 0.311, + "tokens_per_second": 8.16, + "ttft_s": 0.318, "response_preview": "on-topic " }, { @@ -633,30 +633,30 @@ "id": "nvidia/llama-3.1-nemotron-nano-8b-v1", "label": "Llama 3.1 Nemotron Nano 8B V1", "status": "ok", - "total_time_s": 8.896, + "total_time_s": 7.291, "tokens": 8, - "tokens_per_second": 0.9, - "ttft_s": 1.531, + "tokens_per_second": 1.1, + "ttft_s": 1.296, "response_preview": " The capital of India is New Delhi." }, { "id": "nvidia/llama-3.1-nemotron-nano-vl-8b-v1", "label": "Llama 3.1 Nemotron Nano VL 8B V1", "status": "ok", - "total_time_s": 0.531, + "total_time_s": 0.5, "tokens": 8, - "tokens_per_second": 15.07, - "ttft_s": 0.335, + "tokens_per_second": 16.0, + "ttft_s": 0.312, "response_preview": "New Delhi is the capital of India." }, { "id": "nvidia/llama-3.1-nemotron-safety-guard-8b-v3", "label": "Llama 3.1 Nemotron Safety Guard 8B V3", "status": "ok", - "total_time_s": 0.496, + "total_time_s": 0.479, "tokens": 8, - "tokens_per_second": 16.14, - "ttft_s": 0.346, + "tokens_per_second": 16.7, + "ttft_s": 0.331, "response_preview": "{\"User Safety\": \"safe\"} " }, { @@ -673,20 +673,20 @@ "id": "nvidia/llama-3.3-nemotron-super-49b-v1", "label": "Llama 3.3 Nemotron Super 49B V1", "status": "ok", - "total_time_s": 0.846, + "total_time_s": 0.717, "tokens": 2, - "tokens_per_second": 2.36, - "ttft_s": 0.437, + "tokens_per_second": 2.79, + "ttft_s": 0.468, "response_preview": "The capital of India is New Delhi." }, { "id": "nvidia/llama-3.3-nemotron-super-49b-v1.5", "label": "Llama 3.3 Nemotron Super 49B V1.5", "status": "ok", - "total_time_s": 9.406, + "total_time_s": 8.991, "tokens": 2, - "tokens_per_second": 0.21, - "ttft_s": 9.304, + "tokens_per_second": 0.22, + "ttft_s": 8.82, "response_preview": "The capital of India is New Delhi." }, { @@ -713,41 +713,41 @@ "id": "nvidia/nemotron-3-content-safety", "label": "Nemotron 3 Content Safety", "status": "ok", - "total_time_s": 0.684, + "total_time_s": 1.219, "tokens": 4, - "tokens_per_second": 5.85, - "ttft_s": 0.682, + "tokens_per_second": 3.28, + "ttft_s": 1.218, "response_preview": "User Safety: safe" }, { "id": "nvidia/nemotron-3-nano-30b-a3b", "label": "Nemotron 3 Nano 30B A3B", "status": "ok", - "total_time_s": 0.589, - "tokens": 2, - "tokens_per_second": 3.39, - "ttft_s": 0.547, + "total_time_s": 0.53, + "tokens": 1, + "tokens_per_second": 1.89, + "ttft_s": 0.528, "response_preview": "The capital of India is New\u202fDelhi." }, { "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "label": "Nemotron 3 Nano Omni 30B A3B Reasoning", "status": "ok", - "total_time_s": 0.571, + "total_time_s": 1.212, "tokens": 2, - "tokens_per_second": 3.51, - "ttft_s": 0.543, + "tokens_per_second": 1.65, + "ttft_s": 1.16, "response_preview": "The capital of India is New\u202fDelhi." }, { "id": "nvidia/nemotron-3-super-120b-a12b", "label": "Nemotron 3 Super 120B A12B", "status": "ok", - "total_time_s": 0.968, + "total_time_s": 3.834, "tokens": 1, - "tokens_per_second": 1.03, - "ttft_s": 0.968, - "response_preview": "The capital of India is New\u202fDelhi." + "tokens_per_second": 0.26, + "ttft_s": 3.834, + "response_preview": "The capital of India is New Delhi." }, { "id": "nvidia/nemotron-4-340b-instruct", @@ -773,30 +773,30 @@ "id": "nvidia/nemotron-content-safety-reasoning-4b", "label": "Nemotron Content Safety Reasoning 4B", "status": "ok", - "total_time_s": 0.424, + "total_time_s": 0.399, "tokens": 8, - "tokens_per_second": 18.88, - "ttft_s": 0.309, + "tokens_per_second": 20.03, + "ttft_s": 0.292, "response_preview": "The capital of India is New Delhi." }, { "id": "nvidia/nemotron-mini-4b-instruct", "label": "Nemotron Mini 4B Instruct", "status": "ok", - "total_time_s": 0.406, + "total_time_s": 0.4, "tokens": 8, - "tokens_per_second": 19.71, - "ttft_s": 0.299, + "tokens_per_second": 19.98, + "ttft_s": 0.289, "response_preview": "The capital of India is New Delhi." }, { "id": "nvidia/nemotron-nano-12b-v2-vl", "label": "Nemotron Nano 12B V2 VL", "status": "ok", - "total_time_s": 4.095, + "total_time_s": 0.526, "tokens": 8, - "tokens_per_second": 1.95, - "ttft_s": 3.899, + "tokens_per_second": 15.2, + "ttft_s": 0.344, "response_preview": "The capital of India is New Delhi.\n" }, { @@ -823,10 +823,10 @@ "id": "nvidia/nvidia-nemotron-nano-9b-v2", "label": "Nvidia Nemotron Nano 9B V2", "status": "ok", - "total_time_s": 4.932, + "total_time_s": 6.207, "tokens": 2, - "tokens_per_second": 0.41, - "ttft_s": 4.851, + "tokens_per_second": 0.32, + "ttft_s": 6.095, "response_preview": "\n\nThe capital of India is New Delhi.\n" }, { @@ -843,10 +843,10 @@ "id": "nvidia/riva-translate-4b-instruct-v1.1", "label": "Riva Translate 4B Instruct V1.1", "status": "ok", - "total_time_s": 0.437, + "total_time_s": 0.592, "tokens": 13, - "tokens_per_second": 29.72, - "ttft_s": 0.288, + "tokens_per_second": 21.97, + "ttft_s": 0.308, "response_preview": "What is the capital of India? Please answer in one sentence." }, { @@ -863,21 +863,21 @@ "id": "openai/gpt-oss-120b", "label": "Gpt Oss 120B", "status": "ok", - "total_time_s": 1.658, + "total_time_s": 0.86, "tokens": 9, - "tokens_per_second": 5.43, - "ttft_s": 1.6, + "tokens_per_second": 10.46, + "ttft_s": 0.79, "response_preview": "The capital of India is New\u202fDelhi." }, { "id": "openai/gpt-oss-20b", "label": "Gpt Oss 20B", "status": "ok", - "total_time_s": 1.058, - "tokens": 4, - "tokens_per_second": 3.78, - "ttft_s": 1.007, - "response_preview": "The capital of India is New Delhi." + "total_time_s": 0.694, + "tokens": 9, + "tokens_per_second": 12.97, + "ttft_s": 0.571, + "response_preview": "The capital of India is New\u202fDelhi." }, { "id": "qwen/qwen3-coder-480b-a35b-instruct", @@ -893,80 +893,80 @@ "id": "qwen/qwen3-next-80b-a3b-instruct", "label": "Qwen3 Next 80B A3B Instruct", "status": "ok", - "total_time_s": 4.68, + "total_time_s": 1.07, "tokens": 2, - "tokens_per_second": 0.43, - "ttft_s": 4.569, + "tokens_per_second": 1.87, + "ttft_s": 0.944, "response_preview": "The capital of India is New Delhi." }, { "id": "qwen/qwen3-next-80b-a3b-thinking", "label": "Qwen3 Next 80B A3B Thinking", "status": "ok", - "total_time_s": 2.389, + "total_time_s": 2.268, "tokens": 2, - "tokens_per_second": 0.84, - "ttft_s": 2.295, + "tokens_per_second": 0.88, + "ttft_s": 2.212, "response_preview": "\n\nThe capital of India is New Delhi." }, { "id": "qwen/qwen3.5-122b-a10b", "label": "Qwen3.5 122B A10B", - "status": "error", - "error": "Request timed out.", - "total_time_s": null, + "status": "ok", + "total_time_s": 9.336, "tokens": 0, - "tokens_per_second": 0, - "ttft_s": null + "tokens_per_second": 0.0, + "ttft_s": null, + "response_preview": "" }, { "id": "qwen/qwen3.5-397b-a17b", "label": "Qwen3.5 397B A17B", "status": "ok", - "total_time_s": 134.576, + "total_time_s": 157.671, "tokens": 2, "tokens_per_second": 0.01, - "ttft_s": 134.458, + "ttft_s": 156.81, "response_preview": "The capital of India is New Delhi." }, { "id": "sarvamai/sarvam-m", "label": "Sarvam M", "status": "ok", - "total_time_s": 4.029, - "tokens": 119, - "tokens_per_second": 29.53, - "ttft_s": 0.336, - "response_preview": "Okay, the user is asking for the capital of India. Let me make sure I have the correct information. I remember that Indi" + "total_time_s": 5.562, + "tokens": 165, + "tokens_per_second": 29.67, + "ttft_s": 0.33, + "response_preview": "Okay, the user is asking for the capital of India. Let me make sure I get this right. I remember that India's capital wa" }, { "id": "stepfun-ai/step-3.5-flash", "label": "Step 3.5 Flash", "status": "ok", - "total_time_s": 4.738, - "tokens": 2, - "tokens_per_second": 0.42, - "ttft_s": 4.706, + "total_time_s": 5.258, + "tokens": 1, + "tokens_per_second": 0.19, + "ttft_s": 5.24, "response_preview": "The capital of India is New Delhi." }, { "id": "stockmark/stockmark-2-100b-instruct", "label": "Stockmark 2 100B Instruct", "status": "ok", - "total_time_s": 0.67, + "total_time_s": 0.668, "tokens": 8, - "tokens_per_second": 11.94, - "ttft_s": 0.369, + "tokens_per_second": 11.97, + "ttft_s": 0.363, "response_preview": "The capital of India is New Delhi." }, { "id": "upstage/solar-10.7b-instruct", "label": "Solar 10.7B Instruct", "status": "ok", - "total_time_s": 0.58, + "total_time_s": 0.555, "tokens": 8, - "tokens_per_second": 13.8, - "ttft_s": 0.366, + "tokens_per_second": 14.41, + "ttft_s": 0.334, "response_preview": "The capital of India is New Delhi." }, { @@ -1013,21 +1013,21 @@ "id": "z-ai/glm-5.1", "label": "Glm 5.1", "status": "ok", - "total_time_s": 6.025, + "total_time_s": 156.852, "tokens": 2, - "tokens_per_second": 0.33, - "ttft_s": 5.887, + "tokens_per_second": 0.01, + "ttft_s": 154.179, "response_preview": "The capital of India is New Delhi." }, { "id": "z-ai/glm4.7", "label": "Glm4.7", - "status": "ok", - "total_time_s": 79.479, - "tokens": 3, - "tokens_per_second": 0.04, - "ttft_s": 79.125, - "response_preview": "The capital of India is New Delhi." + "status": "error", + "error": "Request timed out.", + "total_time_s": null, + "tokens": 0, + "tokens_per_second": 0, + "ttft_s": null }, { "id": "z-ai/glm5",