{ "abacusai/dracarys-llama-3.1-70b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Abacus.AI coding fine-tune of Llama 3.1 70B Instruct", "family": "llama", "license": "Llama 3 Community License", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/abacusai/Dracarys-Llama-3.1-70B-Instruct" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Dracarys Llama 3.1 70B Instruct", "open_weights": true, "release_date": "2024-08-14", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct" } ] }, "ai21/jamba-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "AI21's hybrid SSM-Transformer long-context model for enterprise agents and grounded generation", "family": "jamba", "knowledge": "2024-08-22", "limit": { "context": 256000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 94, "sources": [ "https://huggingface.co/ai21labs/AI21-Jamba-Large-1.7" ], "total_billion": 398 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Jamba Large", "open_weights": true, "release_date": "2025-07-03", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ai21labs/AI21-Jamba-Large-1.7" } ] }, "ai21/jamba-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "AI21's efficient, lightweight hybrid SSM-Transformer model for a wide range of tasks", "family": "jamba", "knowledge": "2024-08-22", "limit": { "context": 256000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "sources": [ "https://huggingface.co/ai21labs/AI21-Jamba2-Mini" ], "total_billion": 52 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Jamba Mini", "open_weights": true, "release_date": "2026-01-08", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ai21labs/AI21-Jamba2-Mini" } ] }, "aion-labs/aion-2.0": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aion-2.0", "open_weights": false, "release_date": "2025-12-21", "schema": 2 }, "aion-labs/aion-3.0": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aion-3.0", "open_weights": false, "release_date": "2026-05-05", "schema": 2 }, "aion-labs/aion-3.0-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aion-3.0-Mini", "open_weights": false, "release_date": "2026-05-14", "schema": 2 }, "aion-labs/aion-rp-llama-3.1-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "llama", "knowledge": "2023-12-31", "limit": { "context": 32768, "output": 29491 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/aion-labs/Aion-RP-Llama-3.1-8B" ], "total_billion": 8.030261248 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aion-RP 1.0 (8B)", "open_weights": true, "release_date": "2025-02-04", "schema": 2 }, "aisingapore/gemma-sea-lion-v4-27b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Gemma 3 27B tuned by AI Singapore for Southeast Asian languages and instruction following", "family": "gemma", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete multimodal checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/aisingapore/Gemma-SEA-LION-v4-27B-IT" ], "total_billion": 27.43240664 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Gemma-SEA-LION-v4-27B-IT", "open_weights": true, "release_date": "2025-09-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/aisingapore/Gemma-SEA-LION-v4-27B-IT" } ] }, "aleph-alpha/kolibri-1": { "benchmarks": [ { "dataset": "English text-only questions; denominator and dataset snapshot not stated", "effort": "High", "harness": "eval-framework", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 21.5, "source": "https://huggingface.co/Aleph-Alpha/Kolibri-1", "variant": "English", "version": "unspecified" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Aleph Alpha's German and English mixture-of-experts reasoning model (78B total, 3.46B active) for chat, RAG and agents", "knowledge": "2026-06-18", "license": "Apache-2.0", "limit": { "context": 1048576 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.45757312, "notes": "Exact counts explicitly reported in the model card, converted to billions.", "sources": [ "https://huggingface.co/Aleph-Alpha/Kolibri-1" ], "total_billion": 78.10307456 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kolibri 1", "open_weights": true, "release_date": "2026-10-03", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Aleph-Alpha/Kolibri-1" } ] }, "alibaba/gte-large-en-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English long-context text embedding model producing 1024-dimensional vectors", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/Alibaba-NLP/gte-large-en-v1.5" ], "total_billion": 0.434139136 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "GTE Large EN v1.5", "open_weights": true, "release_date": "2024-04-20", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Alibaba-NLP/gte-large-en-v1.5" } ] }, "alibaba/happyhorse-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text, image and reference to video with realistic motion", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp" ], "output": [ "video/mp4" ] }, "name": "HappyHorse-1.0", "open_weights": false, "release_date": "2026-04-22", "schema": 2, "type": "video", "v2_only": true }, "alibaba/happyhorse-1.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text, image and reference to video with smoother motion and stronger consistency", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp" ], "output": [ "video/mp4" ] }, "name": "HappyHorse-1.1", "open_weights": false, "release_date": "2026-06-16", "schema": 2, "type": "video", "v2_only": true }, "alibaba/qwen-audio-3.0-asr-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Hosted speech recognition model with dialect and 30-language support for audio-to-text transcription up to 5 minutes", "family": "qwen", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/aac", "audio/flac", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/wav" ], "output": [ "text" ] }, "name": "Qwen-Audio-3.0-ASR Flash", "open_weights": false, "release_date": "2026-06", "schema": 2, "type": "transcription", "v2_only": true }, "alibaba/qwen-audio-3.0-tts-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast hosted text-to-speech model for real-time use", "family": "qwen", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Qwen-Audio-3.0-TTS Flash", "open_weights": false, "release_date": "2026-07", "schema": 2, "type": "speech", "v2_only": true }, "alibaba/qwen-audio-3.0-tts-plus": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Quality-first hosted text-to-speech model", "family": "qwen", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Qwen-Audio-3.0-TTS Plus", "open_weights": false, "release_date": "2026-07", "schema": 2, "type": "speech", "v2_only": true }, "alibaba/qwen-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Qwen model for fast chat, extraction, and high-volume workloads", "family": "qwen", "knowledge": "2024-04", "limit": { "context": 1000000, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen Flash", "open_weights": false, "release_date": "2025-07-28", "schema": 2 }, "alibaba/qwen-image": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight Qwen image model for prompt-driven generation and visual design workflows", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image/png" ] }, "name": "Qwen-Image", "open_weights": true, "release_date": "2025-08-19", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen-Image" } ] }, "alibaba/qwen-image-2.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Unified text-to-image generation and image editing model with native RGBA transparency and reference-image editing", "family": "qwen", "last_updated": "2026-09-21", "license": "qwen-research", "limit": { "context": 8192, "output": 0 }, "links": [ { "label": "Blog", "type": "announcement", "url": "https://qwen.ai/blog?id=qwen-image-2.1" }, { "label": "GitHub", "type": "docs", "url": "https://github.com/QwenLM/Qwen-Image-2.1" } ], "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Visual generation component", "notes": "Official 32-layer Single-Stream DiT specification; excludes the text encoder and other pipeline components.", "sources": [ "https://huggingface.co/Qwen/Qwen-Image-2.1" ], "total_billion": 7 } ], "notes": "Only the visual generation component's size is disclosed; no complete pipeline total inferred." } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/tiff", "image/webp", "image/gif" ], "output": [ "image/png" ] }, "name": "Qwen-Image-2.1", "open_weights": true, "release_date": "2026-09-14", "schema": 2, "type": "image", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen-Image-2.1" } ] }, "alibaba/qwen-image-2.1-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Qwen-Image-2.1 API tier for image generation and editing with native transparency", "family": "qwen", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/tiff", "image/webp", "image/gif" ], "output": [ "image/png" ] }, "name": "Qwen-Image-2.1-Pro", "open_weights": false, "release_date": "2026-10-02", "schema": 2, "type": "image", "v2_only": true }, "alibaba/qwen-image-2.1-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "8-step Qwen-Image-2.1 model for fast image generation and editing", "family": "qwen", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Visual generation component", "notes": "Official visual generation architecture count, not the complete pipeline.", "sources": [ "https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo" ], "total_billion": 7 } ], "notes": "Complete pipeline total is unknown; the Turbo card explicitly specifies the same visual generation architecture." } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/tiff", "image/webp", "image/gif" ], "output": [ "image/png" ] }, "name": "Qwen-Image-2.1-Turbo", "open_weights": true, "release_date": "2026-10-09", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo" } ] }, "alibaba/qwen-image-3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Unified image generation and editing model with dense layouts and small-text rendering", "family": "qwen", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/tiff", "image/webp", "image/gif" ], "output": [ "image/png" ] }, "name": "Qwen-Image-3.0", "open_weights": false, "release_date": "2026-08-05", "schema": 2, "type": "image", "v2_only": true }, "alibaba/qwen-image-3.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-quality Qwen-Image-3.0 for complex layouts, small text and photographic detail", "family": "qwen", "limit": { "context": 0, "input": 4500, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/tiff", "image/webp", "image/gif" ], "output": [ "image/png" ] }, "name": "Qwen-Image-3.0-Pro", "open_weights": false, "release_date": "2026-08-05", "schema": 2, "type": "image", "v2_only": true }, "alibaba/qwen-max": { "benchmarks": [ { "date": "2025-01-28", "metric": "percent correct", "name": "Aider Polyglot", "score": 21.8, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Qwen model for complex reasoning, coding, and agentic workflows", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-01-25", "limit": { "context": 32768, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen Max", "open_weights": false, "release_date": "2024-04-03", "schema": 2 }, "alibaba/qwen-mt-plus": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Flagship Qwen translation model with terminology control and support for 92 languages", "family": "qwen", "limit": { "context": 16384, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen MT Plus", "open_weights": false, "release_date": "2025-07-24", "schema": 2 }, "alibaba/qwen-omni-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen omni model for text, vision, audio, and multimodal agent tasks", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-03-26", "limit": { "context": 32768, "output": 2048 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "audio/amr", "audio/wav", "audio/3gpp", "audio/aac", "audio/mpeg", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text", "audio/wav" ] }, "name": "Qwen-Omni Turbo", "open_weights": false, "release_date": "2025-01-19", "schema": 2 }, "alibaba/qwen-plus": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen instruction model for multilingual chat, reasoning, and tool use", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-09-11", "limit": { "context": 1000000, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen Plus", "open_weights": false, "release_date": "2024-01-25", "schema": 2 }, "alibaba/qwen-turbo": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Qwen model for fast chat, extraction, and high-volume workloads", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-04-28", "limit": { "context": 1000000, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen Turbo", "open_weights": false, "release_date": "2024-11-01", "schema": 2 }, "alibaba/qwen-vl-max": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-08-13", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen-VL Max", "open_weights": false, "release_date": "2024-04-08", "schema": 2 }, "alibaba/qwen-vl-plus": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2024-04", "last_updated": "2025-08-15", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen-VL Plus", "open_weights": false, "release_date": "2024-01-25", "schema": 2 }, "alibaba/qwen2-5-vl-72b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2024-04", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not just the language backbone.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen2.5-VL-72B-Instruct" ], "total_billion": 73.410777344 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen2.5-VL 72B Instruct", "open_weights": true, "release_date": "2025-01-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct" } ] }, "alibaba/qwen2.5-7b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Small open Qwen2.5 instruction model for chat, structured output, and tool use", "family": "qwen", "license": "Apache 2.0", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" ], "total_billion": 7.61 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen2.5 7B Instruct", "open_weights": true, "release_date": "2024-09-19", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" } ] }, "alibaba/qwen2.5-coder-0.5b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Tiny open Qwen code model for lightweight completion and on-device coding", "family": "qwen", "license": "Apache 2.0", "limit": { "context": 32768, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B" ], "total_billion": 0.49 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen2.5-Coder-0.5B", "open_weights": true, "release_date": "2024-11-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B" } ] }, "alibaba/qwen2.5-coder-32b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open coding-focused Qwen model for code generation, repair, and repository reasoning", "family": "qwen", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" ], "total_billion": 32.5 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen2.5-Coder-32B-Instruct", "open_weights": true, "release_date": "2024-11-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" } ] }, "alibaba/qwen3-235b-a22b": { "benchmarks": [ { "date": "2025-05-09", "metric": "percent correct", "name": "Aider Polyglot", "score": 59.6, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 21.41, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Large open Qwen MoE for multilingual reasoning, coding, and tool use", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 22, "sources": [ "https://huggingface.co/Qwen/Qwen3-235B-A22B" ], "total_billion": 235 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 235B-A22B", "open_weights": true, "release_date": "2025-04-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B" } ] }, "alibaba/qwen3-235b-a22b-instruct-2507": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Updated large open Qwen3 MoE instruct model for multilingual chat, coding, and tool use", "family": "qwen", "license": "Apache 2.0", "limit": { "context": 262144, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 22, "sources": [ "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" ], "total_billion": 235 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 235B-A22B Instruct 2507", "open_weights": true, "release_date": "2025-07-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" } ] }, "alibaba/qwen3-235b-a22b-thinking-2507": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Updated large open Qwen3 MoE thinking model for reasoning, math, coding, and tool use", "family": "qwen", "license": "Apache 2.0", "limit": { "context": 262144, "output": 81920 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 22, "sources": [ "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507" ], "total_billion": 235 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 235B-A22B Thinking 2507", "open_weights": true, "release_date": "2025-07-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507" } ] }, "alibaba/qwen3-30b-a3b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Sparse MoE Qwen model with 3B active parameters for efficient chat and reasoning", "family": "qwen", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.3, "sources": [ "https://huggingface.co/Qwen/Qwen3-30B-A3B" ], "total_billion": 30.5 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 30B A3B", "open_weights": true, "release_date": "2025-04-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B" } ] }, "alibaba/qwen3-30b-a3b-instruct-2507": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Updated compact Qwen3 MoE instruct model for multilingual chat, coding, and tool use", "family": "qwen", "license": "Apache 2.0", "limit": { "context": 262144, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.3, "sources": [ "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" ], "total_billion": 30.5 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 30B-A3B Instruct 2507", "open_weights": true, "release_date": "2025-07-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" } ] }, "alibaba/qwen3-32b": { "benchmarks": [ { "date": "2025-05-08", "metric": "percent correct", "name": "Aider Polyglot", "score": 40.0, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Dense open Qwen model for self-hosted chat, reasoning, and coding", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-32B" ], "total_billion": 32.8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 32B", "open_weights": true, "release_date": "2025-04-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-32B" } ] }, "alibaba/qwen3-8b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Dense 8B Qwen3 model with switchable thinking for chat, reasoning, and tool use", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-8B" ], "total_billion": 8.2 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 8B", "open_weights": true, "release_date": "2025-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-8B" } ] }, "alibaba/qwen3-asr-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual speech recognition model covering 30 languages and 22 Chinese dialects", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete ASR checkpoint, not the nominal 0.6B designation.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-ASR-0.6B" ], "total_billion": 0.938008576 } } }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Qwen3 ASR 0.6B", "open_weights": true, "release_date": "2026-01", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-ASR-0.6B" } ] }, "alibaba/qwen3-asr-1.7b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual speech recognition model covering 30 languages and 22 Chinese dialects", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete ASR checkpoint, not the nominal 1.7B designation.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-ASR-1.7B" ], "total_billion": 2.349217408 } } }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Qwen3 ASR 1.7B", "open_weights": true, "release_date": "2026-01", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B" } ] }, "alibaba/qwen3-coder-30b-a3b-instruct": { "benchmarks": [ { "date": "2026-06-02", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 19.4, "source": "https://openrouter.ai/qwen/qwen3-coder-30b-a3b-instruct/benchmarks" }, { "date": "2026-06-02", "metric": "percent correct", "name": "SciCode", "score": 27.8, "source": "https://openrouter.ai/qwen/qwen3-coder-30b-a3b-instruct/benchmarks" }, { "date": "2026-06-02", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 15.2, "source": "https://openrouter.ai/qwen/qwen3-coder-30b-a3b-instruct/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Smaller Qwen coder for efficient local agents and repo-level fixes", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.3, "sources": [ "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct" ], "total_billion": 30.5 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3-Coder 30B-A3B Instruct", "open_weights": true, "release_date": "2025-07-31", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct" } ] }, "alibaba/qwen3-coder-480b-a35b-instruct": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 38.7, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Qwen coding heavyweight for repository reasoning and agentic engineering", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 35, "sources": [ "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct" ], "total_billion": 480 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3-Coder 480B-A35B Instruct", "open_weights": true, "release_date": "2025-07-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct" } ] }, "alibaba/qwen3-coder-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen coding model for software agents, repository edits, and code reasoning", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 Coder Flash", "open_weights": false, "release_date": "2025-07-28", "schema": 2 }, "alibaba/qwen3-coder-next": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight Qwen coding model for agents, repository edits, and multi-turn tool use", "family": "qwen", "knowledge": "2025-09", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3-Coder-Next" ], "total_billion": 80 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 Coder Next", "open_weights": true, "release_date": "2026-02-03", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Coder-Next" } ] }, "alibaba/qwen3-coder-plus": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Hosted Qwen coder for software agents, repo edits, and long-context code", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 Coder Plus", "open_weights": false, "release_date": "2025-07-23", "schema": 2 }, "alibaba/qwen3-embedding-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Compact multilingual text embedding model for retrieval, code search, classification, and clustering", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B" ], "total_billion": 0.6 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Qwen3 Embedding 0.6B", "open_weights": true, "release_date": "2025-06-05", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B" } ] }, "alibaba/qwen3-embedding-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text embedding model for retrieval, code search, classification, and clustering", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-Embedding-4B" ], "total_billion": 4 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Qwen3 Embedding 4B", "open_weights": true, "release_date": "2025-06-05", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Embedding-4B" } ] }, "alibaba/qwen3-embedding-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text embedding model for retrieval, code search, classification, and clustering", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-Embedding-8B" ], "total_billion": 8 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Qwen3 Embedding 8B", "open_weights": true, "release_date": "2025-06-05", "schema": 2, "type": "embedding", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Embedding-8B" } ] }, "alibaba/qwen3-max": { "benchmarks": [ { "date": "2026-05-30", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 26.4, "source": "https://openrouter.ai/qwen/qwen3-max/benchmarks" }, { "date": "2026-05-30", "metric": "percent correct", "name": "SciCode", "score": 38.3, "source": "https://openrouter.ai/qwen/qwen3-max/benchmarks" }, { "date": "2026-05-30", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 20.5, "source": "https://openrouter.ai/qwen/qwen3-max/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Qwen3 model for coding agents, complex reasoning, and tool use", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 Max", "open_weights": false, "release_date": "2025-09-23", "schema": 2 }, "alibaba/qwen3-max-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Qwen model for complex reasoning, coding, and agentic workflows", "family": "qwen", "knowledge": "2025-04", "last_updated": "2025-09-23", "limit": { "context": 262144, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3 Max Preview", "open_weights": false, "release_date": "2025-09-05", "schema": 2 }, "alibaba/qwen3-next-80b-a3b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen instruction model for multilingual chat, reasoning, and tool use", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct" ], "total_billion": 80 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3-Next 80B-A3B Instruct", "open_weights": true, "release_date": "2025-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct" } ] }, "alibaba/qwen3-next-80b-a3b-thinking": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Qwen thinking model for local reasoning, math, and coding agents", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking" ], "total_billion": 80 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3-Next 80B-A3B (Thinking)", "open_weights": true, "release_date": "2025-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking" } ] }, "alibaba/qwen3-omni-30b-a3b-thinking": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal reasoning model accepting text, images, video, and audio with text output", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 65536, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete Thinking checkpoint; not the separate Instruct release with a Talker. Active count not inferred from the name.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-Omni-30B-A3B-Thinking" ], "total_billion": 31.719205488 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv", "audio/amr", "audio/wav", "audio/3gpp", "audio/aac", "audio/mpeg" ], "output": [ "text" ] }, "name": "Qwen3 Omni 30B A3B Thinking", "open_weights": true, "release_date": "2025-09-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking" } ] }, "alibaba/qwen3-reranker-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Compact multilingual query-document reranking model supporting more than 100 languages", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-Reranker-0.6B" ], "total_billion": 0.6 } } }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Qwen3 Reranker 0.6B", "open_weights": true, "release_date": "2025-06-05", "schema": 2, "type": "reranking", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Reranker-0.6B" } ] }, "alibaba/qwen3-reranker-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual query-document reranking model supporting more than 100 languages", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3-Reranker-8B" ], "total_billion": 8 } } }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Qwen3 Reranker 8B", "open_weights": true, "release_date": "2025-06-05", "schema": 2, "type": "reranking", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-Reranker-8B" } ] }, "alibaba/qwen3-vl-235b-a22b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language instruct model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2025-03-31", "limit": { "context": 131072, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint. Active count not inferred from the name.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-VL-235B-A22B-Instruct" ], "total_billion": 235.670022896 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen3 VL 235B A22B Instruct", "open_weights": true, "release_date": "2025-09-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct" } ] }, "alibaba/qwen3-vl-235b-a22b-thinking": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language thinking model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2025-03-31", "limit": { "context": 131072, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint. Active count not inferred from the name.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-VL-235B-A22B-Thinking" ], "total_billion": 235.670022896 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen3 VL 235B A22B Thinking", "open_weights": true, "release_date": "2025-09-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking" } ] }, "alibaba/qwen3-vl-30b-a3b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open vision-language MoE model for image and video understanding, documents, and visual agents", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint. Active count not inferred from the name.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3-VL-30B-A3B-Instruct" ], "total_billion": 31.070754032 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3 VL 30B A3B Instruct", "open_weights": true, "release_date": "2025-10-04", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct" } ] }, "alibaba/qwen3-vl-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "limit": { "context": 262144, "output": 32768 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen3 VL Flash (Alibaba Cloud)", "open_weights": false, "release_date": "2025-10-15", "schema": 2 }, "alibaba/qwen3-vl-plus": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 32768 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic" ], "output": [ "text" ] }, "name": "Qwen3-VL Plus", "open_weights": false, "release_date": "2025-09-23", "schema": 2 }, "alibaba/qwen3.5-0.8b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Smallest Qwen3.5 multimodal model for on-device chat, reasoning, and tool use", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 81920 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-0.8B" ], "total_billion": 0.8 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 0.8B", "open_weights": true, "release_date": "2026-03-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-0.8B" } ] }, "alibaba/qwen3.5-122b-a10b": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 72, "source": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "sources": [ "https://huggingface.co/Qwen/Qwen3.5-122B-A10B" ], "total_billion": 122 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 122B-A10B", "open_weights": true, "release_date": "2026-02-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-122B-A10B" } ] }, "alibaba/qwen3.5-27b": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 72.4, "source": "https://huggingface.co/Qwen/Qwen3.5-27B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-27B" ], "total_billion": 27 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 27B", "open_weights": true, "release_date": "2026-02-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-27B" } ] }, "alibaba/qwen3.5-2b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Qwen3.5 multimodal model for fast chat, reasoning, and tool use", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 81920 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-2B" ], "total_billion": 2 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 2B", "open_weights": true, "release_date": "2026-03-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-2B" } ] }, "alibaba/qwen3.5-35b-a3b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3.5-35B-A3B" ], "total_billion": 35 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 35B-A3B", "open_weights": true, "release_date": "2026-02-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B" } ] }, "alibaba/qwen3.5-35b-a3b-base": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Pre-trained Qwen3.5 base checkpoint for fine-tuning and research, without instruction tuning", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3.5-35B-A3B-Base" ], "total_billion": 35 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 35B-A3B Base", "open_weights": true, "release_date": "2026-02-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B-Base" } ] }, "alibaba/qwen3.5-397b-a17b": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 76.4, "source": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Large open Qwen multimodal MoE for visual agents and long technical tasks", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 17, "sources": [ "https://huggingface.co/Qwen/Qwen3.5-397B-A17B" ], "total_billion": 397 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 397B-A17B", "open_weights": true, "release_date": "2026-02-15", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B" } ] }, "alibaba/qwen3.5-4b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Small Qwen3.5 multimodal model for low-cost chat, reasoning, and tool use", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 81920 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-4B" ], "total_billion": 4 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 4B", "open_weights": true, "release_date": "2026-03-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-4B" } ] }, "alibaba/qwen3.5-9b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen instruction model for multilingual chat, reasoning, and tool use", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-9B" ], "total_billion": 9 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 9B", "open_weights": true, "release_date": "2026-02-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-9B" } ] }, "alibaba/qwen3.5-9b-base": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Pre-trained Qwen3.5 base checkpoint for fine-tuning and research, without instruction tuning", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.5-9B-Base" ], "total_billion": 9 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 9B Base", "open_weights": true, "release_date": "2026-02-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.5-9B-Base" } ] }, "alibaba/qwen3.5-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 Flash", "open_weights": false, "release_date": "2026-02-23", "schema": 2 }, "alibaba/qwen3.5-plus": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.5 Plus", "open_weights": false, "release_date": "2026-02-16", "schema": 2 }, "alibaba/qwen3.6-27b": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 77.2, "source": "https://huggingface.co/Qwen/Qwen3.6-27B" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 24.0, "source": "https://huggingface.co/Qwen/Qwen3.6-27B" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 969, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Non-reasoning", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.6-27B" ], "total_billion": 27 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.6 27B", "open_weights": true, "release_date": "2026-04-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.6-27B" } ] }, "alibaba/qwen3.6-35b-a3b": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 73.4, "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B" }, { "name": "Humanity's Last Exam", "score": 21.4, "source": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/raw/main/README.md" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup agent loop; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": "899 ± 18", "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Reasoning", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup agent loop; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": "854 ± 24", "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Non-reasoning", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal Qwen MoE for local agents that need vision, audio, and code", "family": "qwen", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/Qwen/Qwen3.6-35B-A3B" ], "total_billion": 35 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.6 35B-A3B", "open_weights": true, "release_date": "2026-04-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B" } ] }, "alibaba/qwen3.6-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen3.6", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.6 Flash", "open_weights": false, "release_date": "2026-04-27", "schema": 2 }, "alibaba/qwen3.6-max-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Qwen model for complex reasoning, coding, and agentic workflows", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 262144, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3.6 Max Preview", "open_weights": false, "release_date": "2026-04-20", "schema": 2 }, "alibaba/qwen3.6-plus": { "benchmarks": [ { "dataset": "2,158 text-only questions from the original HLE dataset (Artificial Analysis evaluation)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": "28%", "source": "https://artificialanalysis.ai/models/comparisons/qwen3-6-plus-vs-gemini-3-1-flash-lite-preview" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Earlier Qwen multimodal workhorse for million-token agent and document tasks", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.6 Plus", "open_weights": false, "release_date": "2026-04-02", "schema": 2 }, "alibaba/qwen3.7-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lightweight multimodal Qwen model for high-throughput text, image, and video tasks", "family": "qwen", "limit": { "context": 1000000, "input": 991808, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.7 Flash", "open_weights": false, "release_date": "2026-07-15", "schema": 2 }, "alibaba/qwen3.7-max": { "benchmarks": [ { "date": "2026-05-19", "metric": "resolved", "name": "SWE-Bench Verified", "score": 80.4, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 60.6, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 78.3, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 69.7, "source": "https://qwen.ai/blog?id=qwen3.7", "version": "2.0" }, { "date": "2026-05-19", "metric": "accuracy", "name": "GPQA Diamond", "score": 92.4, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 41.4, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "name": "SciCode", "score": 53.5, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "metric": "success rate", "name": "MCP Atlas", "score": 76.4, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "date": "2026-05-19", "harness": "Claude Code", "name": "NL2Repo", "score": 47.2, "source": "https://qwen.ai/blog?id=qwen3.7" }, { "dataset": "113 tasks", "name": "DeepSWE", "score": 21.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "Qwen reports the higher score across Claude Code and mini-SWE-agent; temperature=1.0; top_p=0.95; 256K context; selected harness not specified", "version": "1.1" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "2% (rounded display)", "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-max-vs-qwen3-7-max", "variant": "pass@1 across three repeats; maximum 500 agent steps; rounded integer display", "version": "4.0.0" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 53.5, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "with tools" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1128, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-7-max-vs-step-3-5-flash-0202", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3.7 Max", "open_weights": false, "release_date": "2026-05-21", "schema": 2 }, "alibaba/qwen3.7-plus": { "benchmarks": [ { "dataset": "100-task Chartography set", "harness": "Inspect AI; Gemini 3.5 Flash judge", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 15.8, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Thinking on; 10 runs per task" }, { "dataset": "May 2025 revision; 2,158 text-only questions", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 36, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-7-plus-vs-glm-5-1", "variant": "Reasoning version; pass@1; no tools" }, { "dataset": "220 tasks; 44 occupations; 9 industries", "harness": "Artificial Analysis Stirrup; shell access and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 768, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-7-plus-vs-glm-5-1", "variant": "Reasoning version", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal Qwen workhorse for long-context agents, visual inputs, and coding", "family": "qwen", "knowledge": "2025-04", "limit": { "context": 1000000, "output": 64000 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.7 Plus", "open_weights": false, "release_date": "2026-06-02", "schema": 2 }, "alibaba/qwen3.7-text-embedding": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual Qwen3.7 text embedding model with 256 to 2,560 dimensions and 128K-token inputs", "family": "qwen", "limit": { "context": 128000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Qwen3.7 Text Embedding", "open_weights": false, "release_date": "2026-07-15", "schema": 2, "type": "embedding", "v2_only": true }, "alibaba/qwen3.7-text-embedding-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lightweight multilingual Qwen3.7 text embedding model for cost and throughput sensitive workloads", "family": "qwen", "limit": { "context": 128000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Qwen3.7 Text Embedding Flash", "open_weights": false, "release_date": "2026-09-01", "schema": 2, "type": "embedding", "v2_only": true }, "alibaba/qwen3.8-2.4t-a95b": { "benchmarks": [ { "dataset": "all 66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 11, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-2-4t-a95b-vs-step-3-7-flash", "variant": "pass@1 averaged over three repeats per task", "version": "4.0.0" }, { "dataset": "2,158 text-only questions from the original 2,500-question HLE set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-2-4t-a95b-vs-step-3-7-flash", "variant": "single answer; text-only subset", "version": "unspecified" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight sparse MoE (2.4T total, 95B active), the open-weight twin of Qwen3.8 Max for coding, research, complex reasoning, and agentic workflows", "family": "qwen", "license": "qwen3.8-max", "limit": { "context": 262144, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 95, "notes": "The official total is 2.4T, converted to B without adding precision.", "sources": [ "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" ], "total_billion": 2400 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Qwen3.8 2.4T A95B", "open_weights": true, "release_date": "2026-08-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" } ] }, "alibaba/qwen3.8-27b": { "benchmarks": [ { "dataset": "default (V2, 642 tasks)", "harness": "Claude Code", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 61.7, "source": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro", "variant": "temperature=1.0; top_p=0.95; context=256K", "version": "2.0.0" }, { "harness": "Terminus", "metric": "score", "name": "Terminal-Bench", "score": 73, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "version": "2.1" }, { "harness": "Claude Code", "metric": "score", "name": "NL2Repo", "score": 42.3, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "harness": "Claude Code", "metric": "score", "name": "DeepSWE", "score": 42.2, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "version": "1.1" }, { "dataset": "internal", "harness": "Claude Code", "metric": "avg@3", "name": "QwenSWEBench", "score": 79, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "dataset": "internal", "metric": "score", "name": "CoWorkBench", "score": 70.7, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "JobBench", "score": 33.4, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "pass@1", "name": "Agents' Last Exam", "score": 20.4, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "Agents' Last Exam", "score": 42.9, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "IFBench", "score": 79.5, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "GPQA Diamond", "score": 89.2, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 30.8, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "without tools; GPT-4o judge" }, { "metric": "score", "name": "LiveCodeBench", "score": 90.3, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "version": "6" }, { "metric": "score", "name": "OSWorld-Verified", "score": 84.3, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "harness": "OSWorld scaffold; official WebArena-Verified grader", "metric": "score", "name": "WebArena-Verified", "score": 64.8, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "AndroidWorld", "score": 81.9, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "SWE-MM", "score": 38.6, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "RealWorldQA", "score": 85.9, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "score", "name": "ERQA", "score": 65.5, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "dataset": "internal", "metric": "score", "name": "RecreationBench", "score": 47.1, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "pass@3", "name": "ClawEval-MM", "score": 57.4, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "metric": "average score", "name": "ClawEval-MM", "score": 56.9, "source": "https://huggingface.co/Qwen/Qwen3.8-27B" }, { "harness": "Claude Code", "metric": "score", "name": "Vision2Web", "score": 62.9, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "gpt-5.4-2026-03-05 judge" }, { "dataset": "corrected annotations", "metric": "score", "name": "MathVision", "score": 90, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "without code interpreter" }, { "dataset": "corrected annotations", "metric": "score", "name": "MathVision", "score": 94.6, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "with code interpreter" }, { "metric": "score", "name": "BabyVision", "score": 65.7, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "without code interpreter" }, { "metric": "score", "name": "BabyVision", "score": 85.6, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "with code interpreter" }, { "dataset": "corrected annotations", "metric": "score", "name": "CharXiv Reasoning", "score": 83.7, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "without code interpreter" }, { "dataset": "corrected annotations", "metric": "score", "name": "CharXiv Reasoning", "score": 90.2, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "variant": "with code interpreter" }, { "metric": "score", "name": "OmniDocBench", "score": 91.1, "source": "https://huggingface.co/Qwen/Qwen3.8-27B", "version": "1.5" }, { "cost": 3.7729435511821583, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 10.0316676028892, "source": "https://artificialanalysis.ai/models/qwen3-8-27b-non-reasoning", "variant": "Non-reasoning", "version": "5.0.1" }, { "cost": 2.1600047179927624, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 17.4831648632067, "source": "https://artificialanalysis.ai/models/qwen3-8-27b-medium", "version": "5.0.1" }, { "cost": 1.9059046111258202, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 15.6850612405479, "source": "https://artificialanalysis.ai/models/qwen3-8-27b-low", "version": "5.0.1" }, { "cost": 3.5295494751410654, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 20.1068756031705, "source": "https://artificialanalysis.ai/models/qwen3-8-27b", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Xhigh", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 33.9, "source": "https://artificialanalysis.ai/models/qwen3-8-27b", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Xhigh", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 15.3, "source": "https://artificialanalysis.ai/models/qwen3-8-27b", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Dense 27B vision-language model for coding, agent tasks, and image and video understanding", "family": "qwen", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/Qwen3.8-27B" ], "total_billion": 27 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.8 27B", "open_weights": true, "release_date": "2026-08-14", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.8-27B" } ] }, "alibaba/qwen3.8-flash": { "benchmarks": [ { "metric": "score", "name": "DeepSWE", "score": 58.7, "source": "https://docs.qwencloud.com/developer-guides/getting-started/latest-model", "version": "1.1" }, { "metric": "score", "name": "SWE-bench Pro", "score": 62.5, "source": "https://docs.qwencloud.com/developer-guides/getting-started/latest-model" }, { "dataset": "100 tasks", "effort": "xHigh reasoning", "metric": "accuracy/mean", "name": "Chartography", "score": 17.4, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen vision-language model for visual reasoning, documents, and agent tasks", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.8 Flash", "open_weights": false, "release_date": "2026-08-26", "schema": 2 }, "alibaba/qwen3.8-flash-next": { "benchmarks": [ { "harness": "mini-swe-agent", "metric": "score", "name": "DeepSWE", "score": 58.7, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "version": "1.1" }, { "dataset": "Qwen refined and corrected task set", "harness": "Claude Code", "metric": "score", "name": "SWE-Bench Pro", "score": 62.5, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "dataset": "66 tasks", "harness": "mini-SWE-agent v2.4.6", "metric": "pass@1", "name": "Terminal-Bench", "score": 25.25, "source": "https://artificialanalysis.ai/models/qwen3-8-flash-next", "variant": "mean over 3 repeats per task; 500-step cap; task timeout up to 8 hours", "version": "4.0.0" }, { "harness": "mini-swe-agent", "metric": "score", "name": "SWE-Bench Multilingual", "score": 81, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "harness": "Claude Code", "metric": "score", "name": "NL2Repo", "score": 48.1, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "dataset": "internal", "metric": "score", "name": "CoWorkBench", "score": 73.9, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "JobBench", "score": 55.7, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "pass@1", "name": "Agents' Last Exam", "score": 24.3, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "Agents' Last Exam", "score": 51.2, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "pass@1", "name": "Toolathlon-Verified", "score": 73.5, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "IFBench", "score": 81.3, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "GPQA Diamond", "score": 91.7, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "Humanity's Last Exam", "score": 35.9, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "GPT-4o judge" }, { "metric": "score", "name": "LiveCodeBench", "score": 91.9, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "version": "6" }, { "metric": "pass@3", "name": "ClawEval-MM", "score": 64.4, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "average score", "name": "ClawEval-MM", "score": 60.4, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "dataset": "internal", "metric": "score", "name": "RecreationBench", "score": 49.9, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "AndroidWorld", "score": 84.5, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "binary completion rate", "name": "OSWorld", "score": 19.4, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "version": "2.0" }, { "metric": "partial score", "name": "OSWorld", "score": 52.3, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "version": "2.0" }, { "harness": "Claude Code", "metric": "score", "name": "Vision2Web", "score": 64, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "gpt-5.4-2026-03-05 judge" }, { "metric": "score", "name": "ERQA", "score": 72.3, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "LVBench", "score": 76.6, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "metric": "score", "name": "RealWorldQA", "score": 88.5, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "dataset": "corrected annotations", "metric": "score", "name": "MathVision", "score": 90.6, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "without code interpreter" }, { "dataset": "corrected annotations", "metric": "score", "name": "MathVision", "score": 95.7, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "with code interpreter" }, { "metric": "score", "name": "CharXiv Reasoning", "score": 84.6, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "without code interpreter" }, { "metric": "score", "name": "CharXiv Reasoning", "score": 90.6, "source": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "variant": "with code interpreter" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight experimental preview of the Qwen4 architecture: hybrid-attention MoE (125B total, 6B active) with vision encoder for coding, agent tasks, and image and video understanding", "family": "qwen", "license": "qwen-community-1.0", "limit": { "context": 262144, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint, excluding separately reported integer tensors. The card's 125B backbone excludes 51B n-gram embeddings and 4B MTP; its 6B active backbone count is not used as a whole-model active count.", "sources": [ "https://huggingface.co/api/models/Qwen/Qwen3.8-Flash-Next", "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" ], "total_billion": 179.999981424 } } }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.8 Flash Next", "open_weights": true, "release_date": "2026-08-27", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" } ] }, "alibaba/qwen3.8-max": { "benchmarks": [ { "harness": "Claude Code", "metric": "avg@10", "name": "Terminal-Bench", "score": 86.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "5h timeout", "version": "2.1" }, { "dataset": "Qwen refined and corrected task set", "harness": "Claude Code", "metric": "resolved", "name": "SWE-Bench Pro", "score": 67.7, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "113 tasks", "harness": "Claude Code", "metric": "pass@1", "name": "DeepSWE", "score": 56.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "temperature=1.0; top_p=0.95; context=256K", "version": "1.1" }, { "harness": "Claude Code", "metric": "score", "name": "NL2Repo", "score": 55.9, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "date": "2026-08-03", "metric": "dominance score", "name": "FrontierSWE", "score": 73.5, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "harness": "Claude Code", "metric": "score", "name": "MLS-Bench-Lite", "score": 41, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "5h timeout" }, { "harness": "BasicAgent", "metric": "score", "name": "PaperBench", "score": 93, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "Code-Dev; 3 runs; 12h timeout; Opus 4.6 judge" }, { "dataset": "95 public tasks", "metric": "avg@3", "name": "AndroidBench", "score": 75.1, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "internal", "harness": "Claude Code", "metric": "avg@3", "name": "QwenSWEBench", "score": 80.7, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "internal", "harness": "Claude Code", "metric": "avg@5", "name": "QwenQoderBench", "score": 58.4, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "internal", "metric": "Elo", "name": "QwenReactBench", "score": 1724, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "internal", "metric": "Elo", "name": "QwenSVGBench", "score": 1713, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "internal", "metric": "score", "name": "CoWorkBench", "score": 74.8, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "WorkSpaceBench", "score": 67.7, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "JobBench", "score": 53.4, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "87 tasks", "harness": "OpenCode", "metric": "avg@3", "name": "SkillsBench", "score": 70.2, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "version": "1.1" }, { "metric": "pass@1", "name": "Agents' Last Exam", "score": 27, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "Agents' Last Exam", "score": 52.4, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "dataset": "600 public tasks", "metric": "pass@1", "name": "AutomationBench", "score": 27.3, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "pass@1", "name": "Toolathlon-Verified", "score": 72.5, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "harness": "Qwen-Agent", "metric": "average item F1", "name": "WideSearch", "score": 81.9, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "Humanity's Last Exam", "score": 56.2, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "with tools" }, { "metric": "score", "name": "Humanity's Last Exam", "score": 43.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "without tools" }, { "metric": "score", "name": "GPQA Diamond", "score": 92.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "IFBench", "score": 82.8, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "expert score", "name": "$OneMillion-Bench", "score": 52.5, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "gemini-3.1-pro-preview judge" }, { "metric": "score", "name": "HealthBench", "score": 60.2, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "metric": "score", "name": "PLawBench", "score": 73.2, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "gemini-3.1-pro-preview judge" }, { "metric": "score", "name": "PRBench", "score": 57.6, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "legal" }, { "metric": "score", "name": "PRBench", "score": 58.3, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "variant": "finance" }, { "dataset": "256K; 8-needle", "metric": "score", "name": "MRCR", "score": 92.9, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "version": "2" }, { "metric": "score", "name": "LongBench", "score": 66.3, "source": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "version": "2" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 19, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-max-vs-qwen3-8-max-0803", "variant": "three repeats; per-task pass@1; averaged result", "version": "4.0.0" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 34.34, "source": "https://www.vals.ai/models/alibaba_qwen3.8-max", "variant": "single run; per-task pass@1; infrastructure/provider errors retried; 8h task limit", "version": "4.0.0" }, { "dataset": "70 tasks; upstream v0.1.0", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 12.86, "source": "https://www.vals.ai/models/alibaba_qwen3.8-max", "variant": "single run; pass@1; infrastructure/provider errors retried; 8h task limit", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "xHigh", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 29.1, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1604, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-qwen3-8-max-0803", "version": "2.1" }, { "cost": 3.7290646500445437, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 57.46102449888641, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 10.633329, "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 34.343, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "2.4-trillion-parameter MoE flagship for coding, professional work, multimodal understanding, and long-horizon agentic workflows", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv", "application/pdf" ], "output": [ "text" ] }, "name": "Qwen3.8 Max", "open_weights": false, "release_date": "2026-08-03", "schema": 2 }, "alibaba/qwen3.8-max-0902": { "benchmarks": [ { "cost": 18.6760733252611, "cost_source": "https://artificialanalysis.ai/models/qwen3-8-max", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "harness": "mini-swe-agent", "metric": "share of tasks passed (pass@1)", "name": "Terminal-Bench", "score": 39, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-max-vs-qwen3-8-max-0803", "variant": "Mean of three pass@1 runs; 500 agent-step limit; upstream task timeouts and sandbox resources", "version": "4.0.0" }, { "cost": 12.146852, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Full (66 tasks)", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 38.889, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Full (70 tasks)", "harness": "mini-swe-agent", "metric": "share of tasks passed (pass@1)", "name": "Terminal-Bench-Science", "score": 12, "source": "https://artificialanalysis.ai/models/comparisons/qwen3-8-max-vs-qwen3-8-max-0803", "variant": "Mean of three pass@1 runs; 1,000 agent-step limit; 30-second timeout per shell command; upstream task timeouts and sandbox resources", "version": "0.1.0" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43, "source": "https://artificialanalysis.ai/models/comparisons/grok-4-6-xhigh-vs-qwen3-8-max" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34, "source": "https://artificialanalysis.ai/models/qwen3-8-max", "variant": "Reasoning version", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1668, "source": "https://artificialanalysis.ai/models/comparisons/grok-4-6-xhigh-vs-qwen3-8-max", "version": "2.1" }, { "cost": 8.729516, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "Six private and two public benchmarks", "metric": "performance score", "name": "Vals Index", "score": 59.045, "source": "https://www.vals.ai/models/alibaba_qwen3.8-max-0902", "variant": "Default provider: Alibaba; temperature=0.7; top-p/top-k default; max output 128000", "version": "2.2" }, { "cost": 10.023891571541945, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 33.6545627367888, "source": "https://artificialanalysis.ai/models/qwen3-8-max", "version": "5.0.1" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 24.0, "source": "https://artificialanalysis.ai/models/qwen3-8-max", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "2026-09-02 upgraded snapshot of Qwen3.8 Max with stronger coding, collaborative agents, and multimodal document understanding", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv", "application/pdf" ], "output": [ "text" ] }, "name": "Qwen3.8 Max 0902", "open_weights": false, "release_date": "2026-09-02", "schema": 2 }, "alibaba/qwen3.8-max-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Preview Qwen flagship for million-token multimodal reasoning and long-horizon agentic workflows", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.8 Max Preview", "open_weights": false, "release_date": "2026-07-19", "schema": 2 }, "alibaba/qwen3.8-max-prime": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "High-throughput edition of Qwen3.8 Max for coding, professional work, multimodal understanding, and long-running agent workflows", "family": "qwen3.8-max", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen 3.8 Max Prime", "open_weights": false, "release_date": "2026-09-23", "schema": 2 }, "alibaba/qwen3.8-omni-flash": { "benchmarks": [ { "harness": "Claude Code / mini-SWE-agent; highest across both, winning harness undisclosed", "metric": "pass@1", "name": "DeepSWE", "score": 57.8, "source": "https://arxiv.org/html/2609.25611", "variant": "temperature=1.0; top_p=0.95; context=256K", "version": "1.1" }, { "dataset": "HLE; tested item count/subset not reported", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 36.5, "source": "https://arxiv.org/html/2609.25611", "variant": "GPT-4o judged", "version": "unspecified" }, { "harness": "Claude Code", "name": "SWE-bench Pro", "score": 63.3, "source": "https://arxiv.org/html/2609.25611", "variant": "temperature=1.0; top_p=0.95; context=256K; problematic tasks corrected and re-evaluated on refined benchmark; benchmark version/task count not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen omni model for text, vision, audio, and multimodal agent tasks", "family": "qwen", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "audio/amr", "audio/wav", "audio/3gpp", "audio/aac", "audio/mpeg", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "Qwen3.8 Omni Flash", "open_weights": false, "release_date": "2026-09-17", "schema": 2 }, "alibaba/qwen4-max-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Preview flagship multimodal model for coding, office productivity, frontend design, and complex agentic tasks", "family": "qwen", "limit": { "context": 1000000, "output": 393216 }, "modalities": { "input": [ "text", "image/bmp", "image/jpeg", "image/png", "image/tiff", "image/webp", "image/heic", "video/mp4", "video/x-msvideo", "video/x-matroska", "video/quicktime", "video/x-flv", "video/x-ms-wmv", "application/pdf" ], "output": [ "text" ] }, "name": "Qwen4 Max Preview", "open_weights": false, "release_date": "2026-11-11", "schema": 2, "v2_only": true }, "alibaba/qwq-32b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open reasoning model from the Qwen team for math, coding, and step-by-step problem solving", "family": "qwen", "knowledge": "2024-04", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/Qwen/QwQ-32B" ], "total_billion": 32.5 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "QwQ 32B", "open_weights": true, "release_date": "2025-03-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Qwen/QwQ-32B" } ] }, "alibaba/qwq-plus": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen reasoning model for deliberate problem solving, math, and coding", "family": "qwen", "knowledge": "2024-04", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "QwQ Plus", "open_weights": false, "release_date": "2025-03-05", "schema": 2 }, "alibaba/text-embedding-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text embedding model covering 50+ languages with 64 to 1,024 dimensions", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Text Embedding v3", "open_weights": false, "release_date": "2024-07-12", "schema": 2, "type": "embedding", "v2_only": true }, "alibaba/text-embedding-v4": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual Qwen3-based text embedding model with 64 to 2,048 dimensions", "family": "qwen", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Text Embedding v4", "open_weights": false, "release_date": "2025-06-05", "schema": 2, "type": "embedding", "v2_only": true }, "alibaba/wan2.6": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text- and image-to-video with audio input and synchronized sound", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Wan2.6", "open_weights": false, "release_date": "2025-12-03", "schema": 2, "type": "video", "v2_only": true }, "alibaba/wan2.7": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text, image and reference to video, with mixed image and video references and voice timbre cloning", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Wan2.7", "open_weights": false, "release_date": "2026-04-03", "schema": 2, "type": "video", "v2_only": true }, "alibaba/wan3.0-video": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "All-in-one video model for reference, editing and image-to-video generation with audio, up to 30 seconds", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Wan3.0", "open_weights": false, "release_date": "2026-08-06", "schema": 2, "type": "video", "v2_only": true }, "alibaba/wan3.0-video-prime": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "High-speed Wan3.0 video model with the same four-modal references, up to 30 seconds", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/bmp", "image/webp", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Wan3.0-Video-Prime", "open_weights": false, "release_date": "2026-08-20", "schema": 2, "type": "video", "v2_only": true }, "allenai/olmocr-2-7b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Vision-language OCR model from Ai2 (Allen Institute for AI) that extracts text from documents where tables, equations, and poor-quality scans defeat traditional OCR", "license": "Apache-2.0", "limit": { "context": 16384 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2510.19817" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "olmOCR 2 7B", "open_weights": true, "release_date": "2025-10-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/allenai/olmOCR-2-7B-1025" } ] }, "amazon/nova-2-lite": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal reasoning model for visual analysis, planning, and tool use", "family": "nova", "knowledge": "2025-10", "last_updated": "2025-12-01", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/x-flv", "video/mpeg", "video/x-ms-wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Nova 2 Lite", "open_weights": false, "release_date": "2025-12-02", "schema": 2 }, "amazon/nova-2-lite-v1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal reasoning model for visual analysis, planning, and tool use", "family": "nova", "knowledge": "2025-10", "limit": { "context": 1000000, "output": 65535 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/x-flv", "video/mpeg", "video/x-ms-wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Nova 2 Lite", "open_weights": false, "release_date": "2025-12-02", "schema": 2 }, "amazon/nova-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "nova-lite", "knowledge": "2024-10", "limit": { "context": 300000, "output": 10000 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/x-flv", "video/mpeg", "video/x-ms-wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Nova Lite", "open_weights": false, "release_date": "2024-12-03", "schema": 2 }, "amazon/nova-lite-v1": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 3.64, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "nova-lite", "knowledge": "2024-10-31", "limit": { "context": 300000, "output": 5120 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "Nova Lite 1.0", "open_weights": false, "release_date": "2024-12-05", "schema": 2 }, "amazon/nova-micro": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "nova-micro", "knowledge": "2024-10", "limit": { "context": 128000, "output": 10000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nova Micro", "open_weights": false, "release_date": "2024-12-03", "schema": 2 }, "amazon/nova-micro-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "nova-micro", "knowledge": "2024-10-31", "limit": { "context": 128000, "output": 5120 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nova Micro 1.0", "open_weights": false, "release_date": "2024-12-05", "schema": 2 }, "amazon/nova-premier": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal model for complex analysis, long-context understanding, tool use, and model distillation", "family": "nova", "knowledge": "2024-10", "limit": { "context": 1000000, "output": 10000 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/x-flv", "video/mpeg", "video/x-ms-wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Nova Premier", "open_weights": false, "release_date": "2025-04-30", "schema": 2 }, "amazon/nova-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship model for demanding analysis, coding, and production agent workflows", "family": "nova-pro", "knowledge": "2024-10", "limit": { "context": 300000, "output": 10000 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/x-flv", "video/mpeg", "video/x-ms-wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Nova Pro", "open_weights": false, "release_date": "2024-12-03", "schema": 2 }, "amazon/nova-pro-v1": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 4.4, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship model for demanding analysis, coding, and production agent workflows", "family": "nova-pro", "knowledge": "2024-10-31", "limit": { "context": 300000, "output": 5120 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "Nova Pro 1.0", "open_weights": false, "release_date": "2024-12-05", "schema": 2 }, "anthracite-org/magnum-v4-72b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight instruction model for adaptable chat and self-hosted production workloads", "knowledge": "2024-06-30", "limit": { "context": 32768, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/anthracite-org/magnum-v4-72b" ], "total_billion": 72.706203648 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Magnum v4 72B", "open_weights": true, "release_date": "2024-10-22", "schema": 2 }, "anthropic/claude-3-5-haiku-20241022": { "benchmarks": [ { "date": "2024-12-21", "metric": "percent correct", "name": "Aider Polyglot", "score": 28.0, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Claude model for responsive assistance, classification, and lightweight agents", "family": "claude-haiku", "knowledge": "2024-07-31", "limit": { "context": 200000, "output": 8192 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Haiku 3.5", "open_weights": false, "release_date": "2024-10-22", "schema": 2 }, "anthropic/claude-3-5-sonnet-20241022": { "benchmarks": [ { "date": "2025-01-17", "metric": "percent correct", "name": "Aider Polyglot", "score": 51.6, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 4.08, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2024-04-30", "limit": { "context": 200000, "output": 8192 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 3.5 v2", "open_weights": false, "release_date": "2024-10-22", "schema": 2 }, "anthropic/claude-3-7-sonnet-20250219": { "benchmarks": [ { "date": "2025-02-24", "metric": "percent correct", "name": "Aider Polyglot", "score": 64.9, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.04, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2024-10-31", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 3.7", "open_weights": false, "release_date": "2025-02-19", "schema": 2 }, "anthropic/claude-3-haiku-20240307": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Legacy model retained for compatibility with older integrations", "family": "claude-haiku", "knowledge": "2023-08-31", "limit": { "context": 200000, "output": 4096 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Haiku 3", "open_weights": false, "release_date": "2024-03-13", "schema": 2 }, "anthropic/claude-fable-5": { "benchmarks": [ { "date": "2026-06-09", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 80.3, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "metric": "resolved", "name": "SWE-Bench Verified", "score": 95, "source": "https://benchlm.ai/benchmarks/sweVerified" }, { "date": "2026-06-09", "metric": "success rate", "name": "Terminal-Bench", "score": 88.0, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "version": "2.1" }, { "date": "2026-06-09", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 59, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "variant": "no tools" }, { "date": "2026-06-09", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 64.5, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "variant": "with tools" }, { "date": "2026-06-09", "metric": "success rate", "name": "OSWorld-Verified", "score": 85, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "dataset": "Diamond", "date": "2026-06-09", "metric": "pass rate", "name": "FrontierCode", "score": 29.3, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "variant": "high effort" }, { "date": "2026-06-09", "metric": "Elo", "name": "GDPval-AA", "score": 1932, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "date": "2026-06-09", "metric": "success rate", "name": "AutomationBench", "score": 17.4, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "cost": 13.414521495535714, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 69.91150442477876, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 7265.01, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "44.5% ± 3.8%", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "version": "4.0.0" }, { "metric": "accuracy", "name": "Terminal-Bench", "score": 42.0, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Claude Code", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 21.4, "source": "https://www.tbench.ai/news/terminal-bench-science-0-1", "variant": "3 independent trials per task", "version": "0.1.0" }, { "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 24.7, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "version": "0.1" }, { "name": "AutomationBench", "score": 17.1, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 57.8, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 63.8, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "with tools" }, { "dataset": "220 tasks across 44 occupations", "effort": "Max", "metric": "Elo rating", "name": "GDPval-AA", "score": "1618 (CI -16/+16)", "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Opus 4.8 Fallback", "version": "2.1" }, { "dataset": "full 100-task released set", "effort": "Adaptive/Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 34.8, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge", "version": "unspecified" }, { "cost": 21.634702092592594, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 69.72477064220185, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 436 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 9.177638214788733, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 68.6046511627907, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 430 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 6.088186528935186, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.36697247706422, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 436 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 3.757873125874126, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 59.58429561200924, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 433 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 30.339756, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 41.414, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 55.5, "source": "https://artificialanalysis.ai/models/claude-fable-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Claude model for creative writing, analysis, and controlled agent workflows", "family": "claude-fable", "knowledge": "2026-01-31", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Fable 5", "open_weights": false, "release_date": "2026-06-09", "schema": 2 }, "anthropic/claude-fable-5-1": { "benchmarks": [ { "dataset": "70 tasks", "effort": "Max", "harness": "Claude Code --bare", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 52.6, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "production safeguards with fallback", "version": "0.1" }, { "dataset": "66 tasks", "effort": "Max", "harness": "Claude Code --bare", "metric": "accuracy", "name": "Terminal-Bench", "score": 55.8, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "production safeguards with fallback", "version": "4.0.0" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1853, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "production safeguards with fallback", "version": "2" }, { "dataset": "August 2026 task release", "metric": "partial score", "name": "OSWorld", "score": 77.9, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "production safeguards; interventions score zero; no fallback", "version": "2.0" }, { "dataset": "August 2026 task release", "metric": "strict success rate", "name": "OSWorld", "score": 41.7, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "production safeguards; interventions score zero; no fallback", "version": "2.0" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 60.9, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "without tools; production safeguards with fallback" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 65, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "with tools; production safeguards with fallback" }, { "dataset": "657 private held-out tasks", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 31.4, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "with Opus 5 fallback on about 260 of 657 tasks; fallback completions count toward score", "version": "1.0.6" }, { "metric": "score", "name": "CursorBench", "score": 73.4, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "variant": "max effort; production safeguards with fallback", "version": "3.2.0" }, { "dataset": "113 long-horizon coding tasks", "effort": "Max", "metric": "pass@1", "name": "DeepSWE", "score": 67.4, "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "variant": "adaptive thinking; average over 5 trials; default sampling settings", "version": "1.1" }, { "dataset": "Full public split: 642 tasks across 11 repositories", "effort": "High", "harness": "Claude Code", "metric": "resolve rate", "name": "SWE-bench Pro", "score": "99.10 ± 0.50", "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "V2" }, { "cost": 6243.5, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "57.9 ± 3.8", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "variant": "95% confidence interval", "version": "4.0.0" }, { "cost": 4872.04, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Xhigh", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "57.9 ± 3.4", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "variant": "95% confidence interval", "version": "4.0.0" }, { "cost": 3985.38, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "High", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "54.5 ± 3.4", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "variant": "95% confidence interval", "version": "4.0.0" }, { "cost": 2832.9, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Medium", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "53.9 ± 3.4", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "variant": "95% confidence interval", "version": "4.0.0" }, { "cost": 2358.72, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Low", "harness": "Claude Code", "metric": "resolution rate", "name": "Terminal-Bench", "score": "43.3 ± 3.6", "source": "https://www.tbench.ai/?version=4.0&efforts=all", "variant": "95% confidence interval", "version": "4.0.0" }, { "cost": 17.175337, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 58.081, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; per-task pass@1", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": "40.00 ± 5.90", "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single pass@1", "version": "0.1.0" }, { "dataset": "657 private held-out tasks", "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 22.37, "source": "https://fireworks.ai/specialized-intelligence-index/benchmarks/workflow-automation/automationbench", "variant": "no fallback; API mode; 50-step cap", "version": "1.0.6" }, { "dataset": "100 tasks", "effort": "Adaptive/Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 46.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "dataset": "100 tasks", "effort": "Max", "metric": "accuracy", "name": "Chartography", "score": 42.6, "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "variant": "without tools; adaptive thinking; mean of 5 runs; Sonnet 4.6 judge" }, { "dataset": "100 tasks", "effort": "Max", "metric": "accuracy", "name": "Chartography", "score": 86.2, "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "variant": "with tools (container and image crop); adaptive thinking; mean of 5 runs; Sonnet 4.6 judge" }, { "dataset": "100 tasks", "effort": "Max", "metric": "accuracy", "name": "Chartography", "score": 44.8, "source": "https://www.anthropic.com/claude-opus-5-5-system-card", "variant": "without tools; adaptive thinking; mean of 5 runs; Gemini 3.5 Flash judge" }, { "dataset": "100 tasks", "effort": "Max", "metric": "accuracy", "name": "Chartography", "score": 88.4, "source": "https://www.anthropic.com/claude-opus-5-5-system-card", "variant": "with tools; adaptive thinking; mean of 5 runs; Gemini 3.5 Flash judge" }, { "dataset": "2,158 text-only questions from the original HLE set", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 59.1, "source": "https://artificialanalysis.ai/evaluations/humanitys-last-exam", "variant": "Default Fallback; Pass@1; GPT-5.6 Luna medium judge" }, { "dataset": "2,158 text-only questions from the original HLE set", "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 58.7, "source": "https://artificialanalysis.ai/evaluations/humanitys-last-exam", "variant": "Default Fallback; Pass@1; GPT-5.6 Luna medium judge" }, { "cost": 12.993997201178466, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 49.1602124158081, "source": "https://artificialanalysis.ai/models/claude-fable-5-1", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 10.847474706140272, "cost_source": "https://artificialanalysis.ai/models/claude-fable-5-1-xhigh", "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 50.0216384248632, "source": "https://artificialanalysis.ai/models/comparisons/claude-fable-5-1-xhigh-vs-claude-fable-5-1", "variant": "Default Fallback", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1763, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; 95% CI -19/+19", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1740, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; 95% CI -19/+19", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1644, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; 95% CI -18/+18", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Medium", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1554, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; 95% CI -18/+18", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Low", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1477, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; 95% CI -18/+18", "version": "2.1" }, { "cost": 4.471135894646431, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 38.683648810734, "source": "https://artificialanalysis.ai/models/claude-fable-5-1-low", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 5.564264690221474, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 43.171597828284, "source": "https://artificialanalysis.ai/models/claude-fable-5-1-medium", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 7.539956160226559, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 47.174988245604, "source": "https://artificialanalysis.ai/models/claude-fable-5-1-high", "variant": "Default Fallback", "version": "5.0.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "xhigh", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.5, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 60.1, "source": "https://artificialanalysis.ai/models/claude-fable-5-1", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Claude model for demanding reasoning and long-horizon agentic work", "family": "claude-fable", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Fable 5.1", "open_weights": false, "release_date": "2026-09-01", "schema": 2 }, "anthropic/claude-fable-5-2": { "benchmarks": [ { "date": "2026-11-09", "effort": "Xhigh", "metric": "score", "name": "FrontierCode", "score": 49.1, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "safeguards on", "version": "2.0" }, { "date": "2026-11-09", "metric": "score", "name": "FrontierSWE", "score": 69.9, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on", "version": "2" }, { "date": "2026-11-09", "metric": "score", "name": "AutomationBench", "score": 51.4, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on" }, { "date": "2026-11-09", "metric": "score", "name": "Terminal-Bench-Science", "score": 74.6, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on", "version": "0.1" }, { "date": "2026-11-09", "metric": "relaxed score", "name": "WANDR", "score": 82.1, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on" }, { "dataset": "offline set", "date": "2026-11-09", "metric": "partial score", "name": "OSWorld", "score": 89.3, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on", "version": "2.1" }, { "date": "2026-11-09", "effort": "Max", "metric": "average success rate", "name": "RoboDojo Sim", "score": 64.9, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "internal run; safeguards on" }, { "date": "2026-11-09", "metric": "rubric score", "name": "AA-Briefcase", "score": 66.1, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on", "version": "1.1" }, { "date": "2026-11-09", "metric": "Elo", "name": "AA-Briefcase", "score": 1785, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; safeguards on", "version": "1.1" }, { "date": "2026-11-09", "effort": "High", "metric": "score", "name": "Remote Labor Index", "score": 41.3, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "safeguards on" }, { "date": "2026-11-09", "metric": "score", "name": "Humanity's Last Exam", "score": 74.2, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "with tools; highest available effort; safeguards on" }, { "date": "2026-11-09", "metric": "score", "name": "Chartography", "score": 73.4, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "no tools; highest available effort; safeguards on" }, { "cost": 3.097241904274546, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 53.9249388799182, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 4.339918517524423, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 54.5454545454545, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 8.776175363262457, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 53.8095238095238, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 1.2783863461849208, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Low", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 67.99749206673704, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "Low", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 59.7312326227989, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "Low", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1545.68, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; shell and web browsing", "version": "2.1" }, { "cost": 3.8026050641787634, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 56.4083157101519, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 5.251012275091596, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 55.050505050505, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 10.98324440927286, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 60.4761904761905, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 1.3316637351754355, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Medium", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 68.82266338432704, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "Medium", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 61.9091751621872, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "Medium", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1591.16, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; shell and web browsing", "version": "2.1" }, { "cost": 9.078414777860381, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 61.8218785440284, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 13.468788577499403, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 59.5959595959596, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 24.2810515829468, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 70, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 1.6912020876465343, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "High", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 71.35975994544773, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 66.1723818350324, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1718.13, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; shell and web browsing", "version": "2.1" }, { "cost": 14.828587632511116, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 62.2167392403357, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 20.8121385936618, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 61.6161616161616, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 31.366492449010273, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 67.6190476190476, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 2.2740566984684927, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Xhigh", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 74.16293165236524, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 68.8600556070436, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1750.65, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; shell and web browsing", "version": "2.1" }, { "cost": 26.53855780398091, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 64.563595756626, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 31.589769302212897, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 59.5959595959596, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 45.205926007936405, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 73.3333333333333, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 4.4483182548807925, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Max", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 76.42233832611831, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 74.2817423540315, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1831.33, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; shell and web browsing", "version": "2.1" }, { "cost": 17.939202, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.DeTlTtu4.json", "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 71.309, "source": "https://www.vals.ai/benchmarks/vals_index", "variant": "published score; server-side Opus 4.8 / Opus 5 fallbacks included", "version": "2.2" }, { "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 69.483, "source": "https://www.vals.ai/_astro/refusal_fallback.DuYWHFGW.js", "variant": "fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun", "version": "2.2" }, { "cost": 4.938442, "cost_source": "https://www.vals.ai/_astro/benchmark_view_finance_index.SoS3fFjr.json", "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Finance Index", "score": 63.583, "source": "https://www.vals.ai/benchmarks/finance_index", "version": "1" }, { "cost": 8.877409, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_index.DoifPxRC.json", "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Legal Index", "score": 69.175, "source": "https://www.vals.ai/benchmarks/legal_index", "variant": "published score; server-side Opus 5 fallback included", "version": "1" }, { "effort": "High", "metric": "performance score", "name": "Vals Legal Index", "score": 69.09, "source": "https://www.vals.ai/_astro/refusal_fallback.Dygvf87S.js", "variant": "fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun", "version": "1" }, { "cost": 8.790492, "cost_source": "https://www.vals.ai/_astro/benchmark_view_science_index.aXAzSjqu.json", "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Science Index", "score": 63.74, "source": "https://www.vals.ai/benchmarks/science_index", "variant": "published score; server-side Opus 5 fallback included", "version": "1.0" }, { "cost": 69.18581, "cost_source": "https://www.vals.ai/_astro/benchmark_view_code_migration_v1_1.B0T5Uk7z.json", "cost_unit": "USD/test", "dataset": "30 CLI source repositories plus 10 COBOL source repositories; repository-weighted overall", "effort": "High", "harness": "Pi", "metric": "hidden-test pass rate", "name": "Code Migration v1.1", "score": 83.265, "source": "https://www.vals.ai/benchmarks/code-migration-v1-1", "variant": "offline sandbox; published score includes server-side Opus 5 fallback; anti-cheat gating", "version": "1.1" }, { "dataset": "30 CLI source repositories plus 10 COBOL source repositories; repository-weighted overall", "effort": "High", "harness": "Pi", "metric": "hidden-test pass rate", "name": "Code Migration v1.1", "score": 73.02, "source": "https://www.vals.ai/_astro/refusal_fallback.CM0jPETe.js", "variant": "offline sandbox; fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun; anti-cheat gating", "version": "1.1" }, { "cost": 6.672375, "cost_source": "https://www.vals.ai/_astro/benchmark_view_emb_v1_1.BepKO0Tl.json", "cost_unit": "USD/test", "effort": "High", "metric": "modeling score", "name": "EMB v1.1", "score": 77.377, "source": "https://www.vals.ai/benchmarks/emb-v1-1", "variant": "Template cell matching and Scratch rubric scoring; GPT-6 Luna Scratch judge; equal category weights; 3.5-hour task limit", "version": "1.1" }, { "cost": 4.833915, "cost_source": "https://www.vals.ai/_astro/benchmark_view_finance_agent_v2_1.6b1PiRZr.json", "cost_unit": "USD/test", "effort": "High", "metric": "partial credit", "name": "Finance Agent v2.1", "score": 69.47, "source": "https://www.vals.ai/benchmarks/fabv2-1", "variant": "dealbreaker-gated, severity-weighted checks; three-judge jury; two-hour task limit", "version": "2.1" }, { "cost": 0.638668, "cost_source": "https://www.vals.ai/_astro/benchmark_view_immigration.CuSr5RWh.json", "cost_unit": "USD/test", "effort": "High", "metric": "average answer accuracy", "name": "Immigration", "score": 73.982, "source": "https://www.vals.ai/benchmarks/immigration", "variant": "up to five exchanges; GPT-6 Luna simulated person; Claude Sonnet 5 rubric grader", "version": "1" }, { "cost": 3.956441, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_research.BeXLzS5J.json", "cost_unit": "USD/test", "dataset": "Private Test split (208 questions)", "effort": "High", "metric": "all-pass accuracy", "name": "Legal Research Bench", "score": 46.154, "source": "https://www.vals.ai/benchmarks/legal_research", "variant": "published score includes server-side Opus 5 fallback; GPT 5.4 rubric judge; three-hour task limit", "version": "1" }, { "dataset": "Private Test split (208 questions)", "effort": "High", "metric": "all-pass accuracy", "name": "Legal Research Bench", "score": 45.673, "source": "https://www.vals.ai/_astro/refusal_fallback.B3dfh3tF.js", "variant": "fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun; GPT 5.4 rubric judge; three-hour task limit", "version": "1" }, { "cost": 1.552812, "cost_source": "https://www.vals.ai/_astro/benchmark_view_mysterymechanism.CLShNtFx.json", "cost_unit": "USD/test", "effort": "High", "metric": "functional recovery accuracy", "name": "MysteryMechanism", "score": 51.802, "source": "https://www.vals.ai/benchmarks/mysterymechanism", "variant": "binary functional equivalence on private structural probes; noise-aware error thresholds", "version": "1" }, { "cost": 0.786101, "cost_source": "https://www.vals.ai/_astro/benchmark_view_proof_bench.CgnEBbDD.json", "cost_unit": "USD/test", "effort": "High", "harness": "ProofBench", "metric": "proof success rate", "name": "ProofBench v1.1", "score": 100, "source": "https://www.vals.ai/benchmarks/proof_bench", "variant": "Lean 4 checker; Mathlib search and Lean execution; no internet", "version": "1.1" }, { "cost": 3.309035, "cost_source": "https://www.vals.ai/_astro/benchmark_view_tax_agent_bench.CoAIMgK7.json", "cost_unit": "USD/test", "dataset": "Private Test split (193 questions)", "effort": "High", "metric": "citation-adjusted all-pass accuracy", "name": "Tax Agent Bench", "score": 32.092, "source": "https://www.vals.ai/benchmarks/tax_agent_bench", "variant": "all-pass times (0.7 + 0.3 times citation quality); GPT 5.4 rubric judge; three-hour task limit", "version": "1" }, { "cost": 22.13226, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vibe_code_bench.DtQxXDu0.json", "cost_unit": "USD/test", "effort": "High", "harness": "OpenHands", "metric": "test pass rate", "name": "Vibe Code Bench v1.1", "score": 93.771, "source": "https://www.vals.ai/benchmarks/vibe-code", "version": "1.1" }, { "cost": 1.681583, "cost_source": "https://www.vals.ai/_astro/benchmark_view_biomysterybench.ccJddhBB.json", "cost_unit": "USD/test", "dataset": "90 tasks", "effort": "High", "harness": "Terminus 2", "metric": "accuracy", "name": "BioMysteryBench", "score": 83.704, "source": "https://www.vals.ai/benchmarks/biomysterybench", "variant": "mean of three full runs; 200-turn cap; four-hour limit; restricted scientific network; GPT-5.6 Terra judge; policy failures score zero", "version": "1" }, { "cost": 19.366751, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_agent_benchmark.BNvE9afF.json", "cost_unit": "USD/test", "dataset": "120 tasks", "effort": "High", "metric": "all-pass rate", "name": "HLAB-Vals-1.1", "score": 5, "source": "https://www.vals.ai/benchmarks/hlab", "variant": "Harvey v1.1 holdout fixes; Vals harness updates; offline; GPT 5.5 and Claude Sonnet 4.6 judges", "version": "1.1" }, { "cost": 46.822029, "cost_source": "https://www.vals.ai/_astro/benchmark_view_programbench.7VHjeIKB.json", "cost_unit": "USD/test", "dataset": "200 public tasks", "effort": "High", "harness": "mini-SWE-agent", "metric": "fully resolved rate", "name": "ProgramBench", "score": 19, "source": "https://www.vals.ai/benchmarks/programbench", "variant": "published score includes server-side Opus 4.8 / Opus 5 fallbacks; offline cleanroom; 1,000 steps; six-hour task limit", "version": "1" }, { "dataset": "200 public tasks", "effort": "High", "harness": "mini-SWE-agent", "metric": "fully resolved rate", "name": "ProgramBench", "score": 18, "source": "https://www.vals.ai/_astro/refusal_fallback.3mxsml3M.js", "variant": "fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun; offline cleanroom; 1,000 steps; six-hour task limit", "version": "1" }, { "cost": 1.839479, "cost_source": "https://www.vals.ai/_astro/benchmark_view_swebench_multimodal.CYd_UmqN.json", "cost_unit": "USD/test", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Multimodal", "score": 62.292, "source": "https://www.vals.ai/benchmarks/swebench-multimodal", "variant": "single run; no internet; issue screenshots downloaded before the run; upstream SWE-bench verifier", "version": "1" }, { "cost": 14.564414, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.D_ldiP5A.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 65.152, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); eight-hour task limit; no step or cost limit; published score includes server-side Opus 4.8 / Opus 5 fallbacks", "version": "4.0.0" }, { "dataset": "66 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 58.586, "source": "https://www.vals.ai/_astro/refusal_fallback.qAdZ62_N.js", "variant": "avg@3 (mean of three pass@1 runs); fallback-assisted tasks scored as failures; post-hoc rescoring, not a rerun", "version": "4.0.0" }, { "cost": 23.13708, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_science.DmI0IAT2.json", "cost_unit": "USD/test", "dataset": "70 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 55.714, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single-run pass@1; infrastructure/provider-error retries only; eight-hour task limit; published score includes server-side Opus 5 fallback", "version": "0.1.0" }, { "cost": 5.617327431446156, "cost_unit": "USD/task", "effort": "Low", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1562.73, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 2.1165804247563167, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Low", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 38.4397474747475, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.013750208863165666, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Low", "metric": "Net Score", "name": "AA-Omniscience", "score": 62.93333333333333, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 1.53269266200902, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Low", "metric": "all-pass rate", "name": "GDP.pdf", "score": 26.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.128247592, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Low", "metric": "accuracy", "name": "HLE-Diamond", "score": 60.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 0.3129632866319219, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Low", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 60, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-low", "variant": "Default Fallback; five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" }, { "cost": 6.835245733157161, "cost_unit": "USD/task", "effort": "Medium", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1622.24, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 2.6864489410815566, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Medium", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 41.2984090909091, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.015341321777060666, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Medium", "metric": "Net Score", "name": "AA-Omniscience", "score": 63.5, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 1.55747228463872, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Medium", "metric": "all-pass rate", "name": "GDP.pdf", "score": 23.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.183118464, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Medium", "metric": "accuracy", "name": "HLE-Diamond", "score": 66.3, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 0.3669312848560007, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Medium", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 61.5, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-medium", "variant": "Default Fallback; five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" }, { "cost": 15.486568536849699, "cost_unit": "USD/task", "effort": "High", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1677.44, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 9.635322308119388, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "High", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 49.1275757575758, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.019553882127697, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "High", "metric": "Net Score", "name": "AA-Omniscience", "score": 64.03333333333333, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 1.6442819737348697, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "High", "metric": "all-pass rate", "name": "GDP.pdf", "score": 21.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.464489248, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "High", "metric": "accuracy", "name": "HLE-Diamond", "score": 73.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 1.049383457348336, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "High", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 73, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-high", "variant": "Default Fallback; five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" }, { "cost": 28.563319210307895, "cost_unit": "USD/task", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1737.78, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 21.41390231508486, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Xhigh", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 47.5810858585859, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.032593153410362334, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Xhigh", "metric": "Net Score", "name": "AA-Omniscience", "score": 64.76666666666667, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 1.8491524518542, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Xhigh", "metric": "all-pass rate", "name": "GDP.pdf", "score": 21.4, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.8755616879999999, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Xhigh", "metric": "accuracy", "name": "HLE-Diamond", "score": 75.8, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 2.4721107765679204, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Xhigh", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 73.5, "source": "https://artificialanalysis.ai/models/claude-fable-5-2-xhigh", "variant": "Default Fallback; five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" }, { "cost": 59.35153486471977, "cost_unit": "USD/task", "effort": "Max", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1785.32, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 43.3740716159139, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Max", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 48.7597222222222, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.21898487481174234, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Max", "metric": "Net Score", "name": "AA-Omniscience", "score": 66.76666666666667, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 2.51789698324489, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Max", "metric": "all-pass rate", "name": "GDP.pdf", "score": 24.2, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 1.733883952, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Max", "metric": "accuracy", "name": "HLE-Diamond", "score": 80.2, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 6.353041121460681, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Max", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 76.5, "source": "https://artificialanalysis.ai/models/claude-fable-5-2", "variant": "Default Fallback; five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Claude model for demanding reasoning and long-horizon agentic work", "family": "claude-fable", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Fable 5.2", "open_weights": false, "release_date": "2026-11-09", "schema": 2 }, "anthropic/claude-haiku-4-5": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 39.45, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "cost": 0.5317769104171033, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 7.43033851624488, "source": "https://artificialanalysis.ai/models/claude-4-5-haiku-reasoning", "variant": "Reasoning", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Claude lane for lightweight agents, office tasks, and responsive chat", "family": "claude-haiku", "knowledge": "2025-02-28", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Haiku 4.5 (latest)", "open_weights": false, "release_date": "2025-10-15", "schema": 2 }, "anthropic/claude-haiku-4-5-20251001": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Claude model for responsive assistance, classification, and lightweight agents", "family": "claude-haiku", "knowledge": "2025-02-28", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Haiku 4.5", "open_weights": false, "release_date": "2025-10-15", "schema": 2 }, "anthropic/claude-haiku-5-5": { "benchmarks": [ { "metric": "Elo rating", "name": "GDPval-AA", "score": 1620, "source": "https://www.anthropic.com/claude-haiku-5-5", "version": "2.1" }, { "date": "2026-10-07", "metric": "Elo", "name": "AA-Briefcase", "score": 1578, "source": "https://www.anthropic.com/claude-haiku-5-5", "version": "1.1" }, { "metric": "partial score", "name": "OSWorld", "score": 72.4, "source": "https://www.anthropic.com/claude-haiku-5-5", "variant": "offline subset", "version": "2.1" }, { "metric": "score", "name": "Humanity's Last Exam", "score": 45.9, "source": "https://www.anthropic.com/claude-haiku-5-5", "variant": "no tools" }, { "metric": "score", "name": "Humanity's Last Exam", "score": 57.4, "source": "https://www.anthropic.com/claude-haiku-5-5", "variant": "with tools" }, { "metric": "score", "name": "Terminal-Bench", "score": 39.2, "source": "https://www.anthropic.com/claude-haiku-5-5", "version": "4.0.0" }, { "dataset": "Main", "date": "2026-10-07", "metric": "score", "name": "FrontierCode", "score": 46.4, "source": "https://www.anthropic.com/claude-haiku-5-5", "version": "1.1" }, { "metric": "score", "name": "Chartography", "score": 46.4, "source": "https://www.anthropic.com/claude-haiku-5-5", "variant": "no tools" }, { "dataset": "V2 Full; 642 tasks across 11 repositories", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": "87.23 ± 2.36", "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "dataset": "657 tasks per effort", "date": "2026-10-05", "effort": "Low", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 6.1, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "variant": "strict scoring; default fallbacks enabled with no fallback calls; 50-step cap", "version": "1.0.6" }, { "dataset": "657 tasks per effort", "date": "2026-10-05", "effort": "Medium", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 8.5, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "variant": "strict scoring; default fallbacks enabled with no fallback calls; 50-step cap", "version": "1.0.6" }, { "dataset": "657 tasks per effort", "date": "2026-10-05", "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 10.5, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "variant": "strict scoring; default fallbacks enabled with no fallback calls; 50-step cap", "version": "1.0.6" }, { "dataset": "657 tasks per effort", "date": "2026-10-05", "effort": "XHigh", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 9.9, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "variant": "strict scoring; default fallbacks enabled with no fallback calls; 50-step cap", "version": "1.0.6" }, { "dataset": "657 tasks per effort", "date": "2026-10-05", "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 12.0, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "variant": "strict scoring; default fallbacks enabled with no fallback calls; 50-step cap", "version": "1.0.6" }, { "cost": 3.117464, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 36.364, "source": "https://vals-ai.com/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": "10.00 ± 3.61", "source": "https://vals-ai.com/benchmarks/terminal-bench-science", "variant": "single pass@1 run per model", "version": "0.1.0" }, { "dataset": "220 tasks across 44 occupations", "effort": "Medium", "harness": "Stirrup shell+web", "metric": "Elo rating", "name": "GDPval-AA", "score": "1283 (-20/+20)", "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 3.17475, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 58.727, "source": "https://www.vals.ai/models/anthropic_claude-haiku-5-5", "version": "2.2" }, { "cost": 0.15856031956679498, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 29.874315467743, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5-high", "version": "5.0.1" }, { "cost": 0.24836110126596478, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34.6387759486628, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5-xhigh", "version": "5.0.1" }, { "cost": 0.043757686381988486, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 18.6180055310996, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5-low", "version": "5.0.1" }, { "cost": 0.41100741536816304, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 36.4845435880142, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5", "version": "5.0.1" }, { "cost": 0.0917645584066771, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 24.6013254343023, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5-medium", "version": "5.0.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 30.12, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.4, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 29.6, "source": "https://artificialanalysis.ai/models/claude-haiku-5-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Fast Claude model for responsive assistance, classification, and lightweight agents", "family": "claude-haiku", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Haiku 5.5", "open_weights": false, "release_date": "2026-10-07", "schema": 2 }, "anthropic/claude-mythos-5": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Restricted Claude model for advanced cybersecurity and biology research workflows", "family": "claude-mythos", "knowledge": "2026-01-31", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Mythos 5", "open_weights": false, "release_date": "2026-06-09", "schema": 2 }, "anthropic/claude-mythos-5-1": { "benchmarks": [ { "dataset": "66 tasks", "effort": "Max", "harness": "Claude Code --bare", "metric": "accuracy", "name": "Terminal-Bench", "score": 60.9, "source": "https://www-cdn.anthropic.com/0339e6a7c5c7b87f5c07798616dc32c215d14235/Claude%20Fable%205.1%20%26%20Claude%20Mythos%205.1%20System%20Card.pdf", "variant": "10 trials per task", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Restricted Claude model for advanced cybersecurity and biology research workflows", "family": "claude-mythos", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Mythos 5.1", "open_weights": false, "release_date": "2026-09-01", "schema": 2 }, "anthropic/claude-mythos-5-2": { "benchmarks": [ { "date": "2026-11-09", "metric": "score", "name": "Terminal-Bench-Science", "score": 75.3, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort", "version": "0.1" }, { "date": "2026-11-09", "metric": "score", "name": "CyScenarioBench", "score": 76.3, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort; lab-specific tasks" }, { "dataset": "1,829 CVEs; October 2026 run", "date": "2026-11-09", "metric": "advisories found", "name": "CVE Rediscovery", "score": 89.6, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "highest available effort" }, { "date": "2026-11-09", "metric": "score", "name": "De novo binder design", "score": 85.1, "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2", "variant": "24 hr; highest available effort" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Restricted-access Claude model for verified cybersecurity and life sciences research organizations", "family": "claude-mythos", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Mythos 5.2", "open_weights": false, "release_date": "2026-11-09", "schema": 2 }, "anthropic/claude-opus-4-0": { "benchmarks": [ { "date": "2025-05-25", "metric": "percent correct", "name": "Aider Polyglot", "score": 72.0, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4 (latest)", "open_weights": false, "release_date": "2025-05-22", "schema": 2 }, "anthropic/claude-opus-4-1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.1 (latest)", "open_weights": false, "release_date": "2025-08-05", "schema": 2 }, "anthropic/claude-opus-4-1-20250805": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 11.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 7.92, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.1", "open_weights": false, "release_date": "2025-08-05", "schema": 2 }, "anthropic/claude-opus-4-20250514": { "benchmarks": [ { "date": "2025-05-25", "metric": "percent correct", "name": "Aider Polyglot", "score": 72.0, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 10.72, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 6.68, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4", "open_weights": false, "release_date": "2025-05-22", "schema": 2 }, "anthropic/claude-opus-4-5": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-05", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.5 (latest)", "open_weights": false, "release_date": "2025-11-24", "schema": 2 }, "anthropic/claude-opus-4-5-20251101": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 45.89, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 25.2, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 14.16, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Claude model for deep reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2025-05", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.5", "open_weights": false, "release_date": "2025-11-24", "schema": 2 }, "anthropic/claude-opus-4-6": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 51.9, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 33.3, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 30, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 35.58, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 36.67, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 36.08, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 51.3, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 71.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 11.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 70.2, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "max", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34.44, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 19.0, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "non-thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "High-end Claude for difficult coding, planning, and slower expert reasoning", "family": "claude-opus", "knowledge": "2025-05-31", "last_updated": "2026-03-13", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.6", "open_weights": false, "release_date": "2026-02-05", "schema": 2 }, "anthropic/claude-opus-4-7": { "benchmarks": [ { "date": "2026-05-28", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 64.3, "source": "https://www.anthropic.com/news/claude-opus-4-8" }, { "date": "2026-05-28", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 66.1, "source": "https://www.anthropic.com/news/claude-opus-4-8", "version": "2.1" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 48.57, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 66.6, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "max" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 81, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "max" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 44.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "max" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 73.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "max", "version": "2.1" }, { "harness": "Cursor CLI", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 61.2, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 78.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 34.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "Terminal-Bench", "score": 70.6, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 59.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 71.7, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 36.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 71.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GPQA Diamond", "score": 94.2, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "HLE full set (2,500 questions)", "effort": "Max", "harness": "Reasoning-only; Claude Opus 4.6 grader", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.9, "source": "https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf", "variant": "no tools; thinking=auto; 1M total-token cap across contexts; no context compaction" }, { "dataset": "HLE full set (2,500 questions)", "effort": "Max", "harness": "Claude Opus 4.6 grader", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.7, "source": "https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf", "variant": "with web search, web fetch, programmatic tool calling, and code execution; thinking=auto; 1M total-token cap across contexts; no context compaction" }, { "date": "2026-04-23", "metric": "success rate", "name": "OSWorld-Verified", "score": 78.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Chartography full task set (100 tasks)", "harness": "Inspect AI Chartography harness; leaderboard configuration: 10 runs per task, Gemini 3.5 Flash judge", "name": "Chartography", "score": 16.5, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Adaptive/Max" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 36.2, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Stronger Opus tier for advanced software work and high-stakes reasoning", "family": "claude-opus", "knowledge": "2026-01-31", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.7", "open_weights": false, "release_date": "2026-04-16", "schema": 2 }, "anthropic/claude-opus-4-8": { "benchmarks": [ { "date": "2026-05-28", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 69.2, "source": "https://www.anthropic.com/news/claude-opus-4-8" }, { "date": "2026-05-28", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 74.6, "source": "https://www.anthropic.com/news/claude-opus-4-8", "version": "2.1" }, { "metric": "resolved", "name": "SWE-Bench Verified", "score": 88.6, "source": "https://benchlm.ai/benchmarks/sweVerified" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 49.8, "source": "https://www-cdn.anthropic.com/0b4915911bb0d19eca5b5ee635c80fef830a37ea.pdf", "variant": "no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 57.9, "source": "https://www-cdn.anthropic.com/0b4915911bb0d19eca5b5ee635c80fef830a37ea.pdf", "variant": "with tools" }, { "date": "2026-06-09", "metric": "success rate", "name": "OSWorld-Verified", "score": 83.4, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, { "dataset": "Diamond", "date": "2026-06-09", "metric": "pass rate", "name": "FrontierCode", "score": 13.4, "source": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "variant": "high effort" }, { "cost": 13.222583593240094, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 58.97435897435898, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 111 tasks attempted; 429 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 6481.26, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Claude Code", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "23.6% ± 3.6%", "source": "https://www.tbench.ai/?filters=%7B%22sets%22:%7B%22agent_display%22:%5B%22Claude+Code%22%5D%7D%7D", "variant": "95% confidence interval", "version": "4.0.0" }, { "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "22% (rounded display)", "source": "https://artificialanalysis.ai/models/comparisons/claude-opus-4-8-vs-claude-opus-4-7", "variant": "pass@1 averaged across three repeats; 500-step limit; rounded integer display", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Claude Code", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 10.5, "source": "https://www.tbench.ai/news/terminal-bench-science-0-1", "variant": "three independent trials per task", "version": "0.1.0" }, { "dataset": "full task set", "effort": "Adaptive/Max", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 15.9, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "cost": 8.006355704138702, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 54.36241610738255, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 447 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 4.281940017699115, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 51.76991150442478, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 3.4438571028761062, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 48.67256637168141, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 2.293369216740577, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 40.79822616407982, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 17.140717, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 23.232, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents", "family": "claude-opus", "knowledge": "2026-01", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 4.8", "open_weights": false, "release_date": "2026-05-28", "schema": 2 }, "anthropic/claude-opus-5": { "benchmarks": [ { "date": "2026-07-24", "metric": "resolved", "name": "SWE-Bench Verified", "score": 96.0, "source": "https://www.anthropic.com/claude-opus-5-system-card" }, { "date": "2026-07-24", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 79.2, "source": "https://www.anthropic.com/claude-opus-5-system-card" }, { "date": "2026-07-24", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 89.5, "source": "https://www.anthropic.com/claude-opus-5-system-card" }, { "date": "2026-07-24", "metric": "resolve rate", "name": "SWE-Bench Multimodal", "score": 59.4, "source": "https://www.anthropic.com/claude-opus-5-system-card" }, { "date": "2026-07-24", "metric": "resolve rate", "name": "DeepSWE", "score": 68.8, "source": "https://www.anthropic.com/claude-opus-5-system-card", "version": "1.1" }, { "dataset": "Main", "date": "2026-07-24", "metric": "mean@5", "name": "FrontierCode", "score": 53.4, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "medium effort", "version": "1.1" }, { "date": "2026-07-24", "harness": "mini-SWE-agent", "metric": "mean reward", "name": "Frontier-Bench", "score": 43.3, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort", "version": "v0.1" }, { "date": "2026-07-24", "metric": "accuracy", "name": "BrowseComp", "score": 90.8, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "single agent" }, { "date": "2026-07-24", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 56.3, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "no tools" }, { "date": "2026-07-24", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 64.7, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "with tools" }, { "date": "2026-07-24", "metric": "F1", "name": "DeepSearchQA", "score": 95.0, "source": "https://www.anthropic.com/news/claude-opus-5", "variant": "max effort" }, { "date": "2026-07-24", "metric": "success rate", "name": "OSWorld", "score": 70.6, "source": "https://www.anthropic.com/claude-opus-5-system-card", "version": "2.0" }, { "date": "2026-07-24", "metric": "Elo", "name": "GDPval-AA", "score": 1861, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort", "version": "v2" }, { "date": "2026-07-24", "metric": "Elo", "name": "AA-Briefcase", "score": 1720, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort" }, { "date": "2026-07-24", "metric": "success rate", "name": "AutomationBench", "score": 26.0, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort" }, { "date": "2026-07-24", "metric": "accuracy", "name": "ARC-AGI-1", "score": 97.5, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort" }, { "date": "2026-07-24", "metric": "accuracy", "name": "ARC-AGI-2", "score": 90.4, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort" }, { "date": "2026-07-24", "metric": "RHAE", "name": "ARC-AGI-3", "score": 30.2, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "high effort" }, { "date": "2026-07-24", "metric": "score", "name": "HealthBench Professional", "score": 59.8, "source": "https://www.anthropic.com/claude-opus-5-system-card", "variant": "max effort" }, { "cost": 11.837583271396396, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.64864864864865, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 444 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "642 tasks (638/642; reported uncertainty ±0.40 percentage points)", "effort": "Xhigh", "harness": "Claude Code", "metric": "Pass@1", "name": "SWE-bench Pro", "score": 99.4, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "cost": 18.596913, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 53.535, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 27.14, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single run; pass@1; retries for provider infrastructure errors", "version": "0.1.0" }, { "dataset": "657 tasks", "date": "2026-09-17 to 2026-09-18", "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 26.9, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "version": "1.0.6" }, { "dataset": "108 tasks", "effort": "Max", "metric": "binary task completion rate", "name": "OSWorld", "score": 44.33, "source": "https://osworld-v2.xlang.ai/", "variant": "batch tool; 500-step budget", "version": "osworld-v2.1" }, { "dataset": "100 tasks", "effort": "Adaptive/Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 27.3, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task" }, { "dataset": "220 tasks", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1730, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 11.821495652976125, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 44.1646231569372, "source": "https://artificialanalysis.ai/models/claude-opus-5", "version": "5.0.1" }, { "cost": 9.07220232326622, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.15436241610739, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 447 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 6.076084489977728, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 72.82850779510024, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 3.28978764541387, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 68.90380313199105, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 447 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 1.6626223919821828, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 58.12917594654788, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.9, "source": "https://artificialanalysis.ai/models/claude-opus-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Strongest Claude Opus model for coding, agents, and professional work", "family": "claude-opus", "knowledge": "2026-05", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 5", "open_weights": false, "release_date": "2026-07-24", "schema": 2 }, "anthropic/claude-opus-5-5": { "benchmarks": [ { "dataset": "113 tasks", "effort": "Max", "metric": "pass@1", "name": "DeepSWE", "score": 74.2, "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf", "variant": "mean of 5 trials", "version": "1.1" }, { "effort": "Xhigh", "harness": "Claude Code --bare", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 66.36, "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf", "variant": "production safeguards with fallback; 5 trials per task (330 runs)", "version": "4.0.0" }, { "cost": 13.198632, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 65.152, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Main", "metric": "score", "name": "FrontierCode", "score": 54.4, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "max effort; production safeguards with fallback", "version": "1.1" }, { "metric": "score", "name": "CursorBench", "score": 57.8, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "max effort; production safeguards with fallback", "version": "4.0" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-17 to 2026-09-18", "effort": "Low", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 23.3, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "variant": "without fallbacks; strict all-assertions scoring; refusals included", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-17 to 2026-09-18", "effort": "Medium", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 28.6, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "variant": "without fallbacks; strict all-assertions scoring; refusals included", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-17 to 2026-09-18", "effort": "High", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 32.0, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "variant": "without fallbacks; strict all-assertions scoring; refusals included", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-17 to 2026-09-18", "effort": "XHigh", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 34.4, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "variant": "without fallbacks; strict all-assertions scoring; refusals included", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-17 to 2026-09-18", "effort": "Max", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 40.0, "source": "https://zapier.com/benchmarks/claude-opus5_5-report", "variant": "without fallbacks; strict all-assertions scoring; refusals included", "version": "1.0.6" }, { "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 42.47, "source": "https://zapier.com/benchmarks", "variant": "Anthropic default fallback routing", "version": "1.0.6" }, { "effort": "Max", "metric": "partial score", "name": "OSWorld", "score": 81.8, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "production safeguards with fallback", "version": "2.1" }, { "effort": "Max", "metric": "score", "name": "Chartography", "score": 89, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "with tools; production safeguards" }, { "dataset": "100 tasks", "effort": "Adaptive/Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 66.3, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "effort": "Max", "metric": "score", "name": "Humanity's Last Exam", "score": 67.7, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "with tools; production safeguards" }, { "dataset": "2,158 text-only questions from the original 2,500-question HLE set", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 61.4, "source": "https://artificialanalysis.ai/evaluations/humanitys-last-exam", "variant": "Pass@1; Default Fallback" }, { "cost": 10.036409719512749, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 59.1060336078904, "source": "https://artificialanalysis.ai/models/claude-opus-5-5", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 6.047194490848774, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 57.4447060170507, "source": "https://artificialanalysis.ai/models/claude-opus-5-5-xhigh", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 3.4646622820973185, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 51.8359958227294, "source": "https://artificialanalysis.ai/models/claude-opus-5-5-high", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 2.4510894376611723, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 48.2081024960691, "source": "https://artificialanalysis.ai/models/claude-opus-5-5-medium", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 0.9755293224458845, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34.752785007759, "source": "https://artificialanalysis.ai/models/claude-opus-5-5-low", "variant": "Default Fallback", "version": "5.0.1" }, { "effort": "Max", "metric": "Elo rating", "name": "GDPval-AA", "score": 1846, "source": "https://www.anthropic.com/claude-opus-5-5", "variant": "production safeguards", "version": "2.1" }, { "dataset": "OpenAI GDPval (220 tasks)", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1866, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; E2B sandbox; blind pairwise comparisons", "version": "2.1" }, { "dataset": "OpenAI GDPval (220 tasks)", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1835, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; E2B sandbox; blind pairwise comparisons", "version": "2.1" }, { "dataset": "OpenAI GDPval (220 tasks)", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1714, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; E2B sandbox; blind pairwise comparisons", "version": "2.1" }, { "dataset": "OpenAI GDPval (220 tasks)", "effort": "Medium", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1593, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; E2B sandbox; blind pairwise comparisons", "version": "2.1" }, { "dataset": "OpenAI GDPval (220 tasks)", "effort": "Low", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1240, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback; E2B sandbox; blind pairwise comparisons", "version": "2.1" }, { "cost": 28.122421, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "Vals Index composite (finance, coding, legal, tax)", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 70.177, "source": "https://www.vals.ai/benchmarks/vals_index", "variant": "Anthropic default provider; 2.68% fallback rate", "version": "2.2" }, { "dataset": "70 tasks", "effort": "Max", "harness": "Claude Code --bare", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 58.7, "source": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf", "variant": "production safeguards with fallback; 10 trials per task (700 runs)", "version": "0.1.0" }, { "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 47.14, "source": "https://www.vals.ai/benchmarks/terminal-bench-science" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 67.9, "source": "https://artificialanalysis.ai/models/claude-opus-5-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Claude model for long-running agentic coding and knowledge work", "family": "claude-opus", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Opus 5.5", "open_weights": false, "release_date": "2026-09-22", "schema": 2 }, "anthropic/claude-sonnet-4-0": { "benchmarks": [ { "date": "2025-05-24", "metric": "percent correct", "name": "Aider Polyglot", "score": 61.3, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 42.7, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 4 (latest)", "open_weights": false, "release_date": "2025-05-22", "schema": 2 }, "anthropic/claude-sonnet-4-20250514": { "benchmarks": [ { "date": "2025-05-24", "metric": "percent correct", "name": "Aider Polyglot", "score": 61.3, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 7.76, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 5.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2025-03-31", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 4", "open_weights": false, "release_date": "2025-05-22", "schema": 2 }, "anthropic/claude-sonnet-4-5": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 43.6, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2025-07-31", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 4.5 (latest)", "open_weights": false, "release_date": "2025-09-29", "schema": 2 }, "anthropic/claude-sonnet-4-5-20250929": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 13.72, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 7.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Claude model for coding, analysis, agent workflows, and cost control", "family": "claude-sonnet", "knowledge": "2025-07-31", "limit": { "context": 200000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 4.5", "open_weights": false, "release_date": "2025-09-29", "schema": 2 }, "anthropic/claude-sonnet-4-6": { "benchmarks": [ { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 31.2, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 32.21, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Claude Code", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 31.76, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 49.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 70.3, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 14.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 63.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "date": "2026-06-30", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 67.0, "source": "https://www.anthropic.com/news/claude-sonnet-5", "version": "2.1" }, { "date": "2026-06-30", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34.6, "source": "https://www.anthropic.com/news/claude-sonnet-5", "variant": "no tools" }, { "date": "2026-06-30", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.8, "source": "https://www.anthropic.com/news/claude-sonnet-5", "variant": "with tools" }, { "date": "2026-06-30", "metric": "success rate", "name": "OSWorld-Verified", "score": 78.5, "source": "https://www.anthropic.com/news/claude-sonnet-5" }, { "cost": 5.5223517987804875, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 29.933481152993345, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Claude workhorse for coding agents, careful analysis, and production cost control", "family": "claude-sonnet", "knowledge": "2025-08-31", "last_updated": "2026-03-13", "limit": { "context": 1000000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 4.6", "open_weights": false, "release_date": "2026-02-17", "schema": 2 }, "anthropic/claude-sonnet-5": { "benchmarks": [ { "date": "2026-06-30", "metric": "resolved", "name": "SWE-Bench Verified", "score": 85.2, "source": "https://www.anthropic.com/news/claude-sonnet-5" }, { "date": "2026-06-30", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 63.2, "source": "https://www.anthropic.com/news/claude-sonnet-5" }, { "date": "2026-06-30", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 78.3, "source": "https://www.anthropic.com/news/claude-sonnet-5" }, { "date": "2026-06-30", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 80.4, "source": "https://www.anthropic.com/news/claude-sonnet-5", "version": "2.1" }, { "date": "2026-06-30", "metric": "success rate", "name": "OSWorld-Verified", "score": 81.2, "source": "https://www.anthropic.com/news/claude-sonnet-5" }, { "date": "2026-06-30", "metric": "accuracy", "name": "BrowseComp", "score": 84.7, "source": "https://www.anthropic.com/news/claude-sonnet-5", "variant": "single agent" }, { "date": "2026-06-30", "metric": "pass rate", "name": "FrontierCode", "score": 38.8, "source": "https://www.anthropic.com/news/claude-sonnet-5", "version": "v1" }, { "cost": 26.399858950791852, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.84615384615384, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 113 tasks attempted; 442 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "642 public tasks", "effort": "Xhigh", "harness": "Claude Code", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 93.15, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "Reported uncertainty ±1.71 percentage points; uncapped cost; up to 250 turns", "version": "2.0.0" }, { "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 14, "source": "https://artificialanalysis.ai/models/comparisons/claude-sonnet-5-vs-claude-sonnet-4-6", "variant": "pass@1; averaged over 3 repeats per task; maximum 500 agent steps; Artificial Analysis run", "version": "4.0.0" }, { "dataset": "2,158 text-only questions", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 41, "source": "https://artificialanalysis.ai/models/comparisons/claude-sonnet-5-vs-claude-sonnet-4-6", "variant": "pass@1; judged by GPT-5.6 Luna (medium)" }, { "dataset": "220 tasks across 44 occupations", "effort": "Non-reasoning, High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1221, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Reported uncertainty −19/+19 Elo; blind pairwise comparisons", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations", "effort": "Medium", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1167, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Reported uncertainty −18/+18 Elo; blind pairwise comparisons", "version": "2.1" }, { "cost": 11.890634540022173, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 49.667405764966745, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 7.425560161197339, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 48.23008849557522, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 4.079036229, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 39.77777777777778, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 450 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 2.1865550832962137, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 30.51224944320713, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 26.04996, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 10.606, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Everyday Claude agent model for coding, planning, browsing, and general work", "family": "claude-sonnet", "knowledge": "2026-01-31", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 5", "open_weights": false, "release_date": "2026-06-30", "schema": 2 }, "anthropic/claude-sonnet-5-5": { "benchmarks": [ { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 70.6, "source": "https://www.anthropic.com/claude-sonnet-5-5", "version": "4.0.0" }, { "dataset": "Main", "date": "2026-09-28", "metric": "score", "name": "FrontierCode", "score": 46.2, "source": "https://www.anthropic.com/claude-sonnet-5-5", "variant": "max effort", "version": "1.1" }, { "date": "2026-09-28", "metric": "score", "name": "CursorBench", "score": 55.5, "source": "https://www.anthropic.com/claude-sonnet-5-5", "version": "4.0" }, { "metric": "Elo rating", "name": "GDPval-AA", "score": 1844, "source": "https://www.anthropic.com/claude-sonnet-5-5", "variant": "pre-release Claude Platform deployment; Anthropic reports a structured-output issue", "version": "2.1" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 64.5, "source": "https://www.anthropic.com/claude-sonnet-5-5", "variant": "with tools" }, { "metric": "partial score", "name": "OSWorld", "score": 80.1, "source": "https://www.anthropic.com/claude-sonnet-5-5", "version": "2.1" }, { "dataset": "V2 Full; 642 tasks across 11 repositories", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": "95.79 ± 1.30", "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "cost": 16.513006, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 64.141, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (three runs, each pass@1)", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 47.14, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1; single run", "version": "0.1.0" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "Between tools", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 18.6, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "Low", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 18.3, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "Medium", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 23.3, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "High", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 29.4, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "XHigh", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 36.8, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "dataset": "657 held-out evaluation tasks", "date": "2026-09-25", "effort": "Max", "harness": "Zapier API; 50-step cap", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 44.75, "source": "https://zapier.com/benchmarks", "variant": "strict scoring; Anthropic default fallbacks", "version": "1.0.6" }, { "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 61.6, "source": "https://www.anthropic.com/claude-sonnet-5-5", "variant": "no tools" }, { "cost": 10.020621343395753, "cost_source": "https://artificialanalysis.ai/models/claude-sonnet-5-5", "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 54.54133407109, "source": "https://artificialanalysis.ai/models/releases/claude-sonnet-5-5/", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 4.091984916321919, "cost_source": "https://artificialanalysis.ai/models/claude-sonnet-5-5-xhigh", "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 49.9240327787242, "source": "https://artificialanalysis.ai/models/releases/claude-sonnet-5-5/", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 1.6361294399784785, "cost_source": "https://artificialanalysis.ai/models/claude-sonnet-5-5-high", "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 41.515109087048, "source": "https://artificialanalysis.ai/models/releases/claude-sonnet-5-5/", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 0.7806560352024481, "cost_source": "https://artificialanalysis.ai/models/claude-sonnet-5-5-medium", "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 30.8195522148431, "source": "https://artificialanalysis.ai/models/releases/claude-sonnet-5-5/", "variant": "Default Fallback", "version": "5.0.1" }, { "cost": 0.5113424463897498, "cost_source": "https://artificialanalysis.ai/models/claude-sonnet-5-5-low", "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 25.649704549789, "source": "https://artificialanalysis.ai/models/releases/claude-sonnet-5-5/", "variant": "Default Fallback", "version": "5.0.1" }, { "effort": "Max", "metric": "Elo rating", "name": "GDPval-AA", "score": 1838, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Default Fallback", "version": "2.1" }, { "cost": 18.69955, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 70.854, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "xhigh", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.12, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 55.0, "source": "https://artificialanalysis.ai/models/claude-sonnet-5-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 53.3, "source": "https://artificialanalysis.ai/models/claude-sonnet-5-5", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "choice": [ "auto", "none" ], "supported": true }, "top_k": false, "top_p": false }, "description": "Fast Claude model for everyday coding, agents, and knowledge work", "family": "claude-sonnet", "knowledge": "2026-06", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "Claude Sonnet 5.5", "open_weights": false, "release_date": "2026-09-28", "schema": 2 }, "apodex/apodex-1.1-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight 35B MoE model for long-horizon research, forecasting, and tool-assisted work", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/apodex/Apodex-1.1-mini" ], "total_billion": 35.951822704 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Apodex 1.1 Mini", "open_weights": true, "release_date": "2026-08-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/apodex/Apodex-1.1-mini" } ] }, "arcee-ai/trinity-large-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lightly post-trained 398B MoE chat model for creative work, long-context prompts, and tool-using agents", "family": "trinity", "last_updated": "2026-05-28", "license": "OpenMDW-1.1", "limit": { "context": 524288, "output": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Preview" }, { "label": "Announcement", "type": "announcement", "url": "https://www.arcee.ai/blog/trinity-large" }, { "label": "License", "type": "license", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Preview/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 13, "notes": "Approximate total and per-token active counts in the architecture table.", "sources": [ "https://huggingface.co/arcee-ai/Trinity-Large-Preview" ], "total_billion": 398 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Trinity Large Preview", "open_weights": true, "release_date": "2026-01-27", "schema": 2, "weights": [ { "format": "safetensors", "label": "Hugging Face", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Preview" } ] }, "arcee-ai/trinity-large-thinking": { "benchmarks": [ { "dataset": "2,158 text-only questions from the original HLE dataset (Artificial Analysis evaluation)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": "16%", "source": "https://artificialanalysis.ai/models/comparisons/trinity-large-thinking-vs-claude-sonnet-4-6-adaptive" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "harness": "Stirrup with shell and web tools", "metric": "Elo rating", "name": "GDPval-AA", "score": 336, "source": "https://artificialanalysis.ai/models/comparisons/trinity-large-thinking-vs-claude-sonnet-4-6-adaptive", "version": "2.1" }, { "dataset": "Terminal-Bench 4.0 (66 tasks; average of three repeats per task)", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "1%", "source": "https://artificialanalysis.ai/models/comparisons/trinity-large-thinking-vs-claude-sonnet-4-6-adaptive", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning-optimized 398B MoE agent model with extended thinking for long-horizon and multi-turn tool use", "family": "trinity", "last_updated": "2026-05-28", "license": "OpenMDW-1.1", "limit": { "context": 524288, "output": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking" }, { "label": "Announcement", "type": "announcement", "url": "https://www.arcee.ai/blog/trinity-large-thinking" }, { "label": "License", "type": "license", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 13, "notes": "Approximate total and per-token active counts in the architecture table.", "sources": [ "https://huggingface.co/arcee-ai/Trinity-Large-Thinking" ], "total_billion": 398 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Trinity Large Thinking", "open_weights": true, "release_date": "2026-04-01", "schema": 2, "weights": [ { "format": "safetensors", "label": "Hugging Face", "url": "https://huggingface.co/arcee-ai/Trinity-Large-Thinking" } ] }, "arcee-ai/trinity-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning-tuned 26B MoE model with 3B active parameters for agents, tools, and multi-step workloads", "family": "trinity", "last_updated": "2026-05-28", "license": "OpenMDW-1.1", "limit": { "context": 131072, "output": 131072 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/arcee-ai/Trinity-Mini" }, { "label": "Announcement", "type": "announcement", "url": "https://www.arcee.ai/blog/the-trinity-manifesto" }, { "label": "License", "type": "license", "url": "https://huggingface.co/arcee-ai/Trinity-Mini/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/arcee-ai/Trinity-Mini" ], "total_billion": 26 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Trinity Mini", "open_weights": true, "release_date": "2025-12-01", "schema": 2, "weights": [ { "format": "safetensors", "label": "Hugging Face", "url": "https://huggingface.co/arcee-ai/Trinity-Mini" } ] }, "arcee-ai/trinity-nano-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Experimental chat-tuned 6B MoE model with 1B active parameters for low-resource chat and instruction following", "family": "trinity", "last_updated": "2026-05-28", "license": "OpenMDW-1.1", "limit": { "context": 131072, "output": 131072 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/arcee-ai/Trinity-Nano-Preview" }, { "label": "Announcement", "type": "announcement", "url": "https://www.arcee.ai/blog/the-trinity-manifesto" }, { "label": "License", "type": "license", "url": "https://huggingface.co/arcee-ai/Trinity-Nano-Preview/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 1, "notes": "Uses the inclusive 1B active count; the separate 800M figure excludes embeddings.", "sources": [ "https://huggingface.co/arcee-ai/Trinity-Nano-Preview" ], "total_billion": 6 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Trinity Nano Preview", "open_weights": true, "release_date": "2025-12-01", "schema": 2, "weights": [ { "format": "safetensors", "label": "Hugging Face", "url": "https://huggingface.co/arcee-ai/Trinity-Nano-Preview" } ] }, "assemblyai/universal-3-5-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with code switching, contextual prompting, and speaker diarization", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Universal-3.5 Pro", "open_weights": false, "release_date": "2026-06-15", "schema": 2, "type": "transcription", "v2_only": true }, "assemblyai/universal-3-6-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model trained on voice-agent and telephony audio, successor to Universal-3.5 Pro", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Universal-3.6 Pro", "open_weights": false, "release_date": "2026-09-29", "schema": 2, "type": "transcription", "v2_only": true }, "assemblyai/universal-3-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Promptable speech-to-text model with natural-language prompting, keyterms and speaker diarization", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Universal-3 Pro", "open_weights": false, "release_date": "2026-02-03", "schema": 2, "type": "transcription", "v2_only": true }, "baai/bge-base-en-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model for retrieval and semantic search", "family": "bge", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes the separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/BAAI/bge-base-en-v1.5" ], "total_billion": 0.10948224 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE Base EN v1.5", "open_weights": true, "release_date": "2023-09-12", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-base-en-v1.5" } ] }, "baai/bge-en-icl": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "LLM-based English embedding model that takes few-shot examples in the query", "family": "bge", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE EN ICL", "open_weights": true, "release_date": "2024-07-25", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-en-icl" } ] }, "baai/bge-large-en-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model for retrieval and semantic search", "family": "bge", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes the separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/BAAI/bge-large-en-v1.5" ], "total_billion": 0.335141888 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE Large EN v1.5", "open_weights": true, "release_date": "2023-09-12", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-large-en-v1.5" } ] }, "baai/bge-m3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual embedding model combining dense, sparse, and multi-vector retrieval", "family": "bge", "license": "MIT", "limit": { "context": 8192, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official BGE documentation's 569M parameter specification, converted to B.", "sources": [ "https://bge-model.com/bge/bge_m3.html" ], "total_billion": 0.569 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE M3", "open_weights": true, "release_date": "2024-01-30", "schema": 2, "type": "embedding", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-m3" } ] }, "baai/bge-multilingual-gemma2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual Gemma 2 based embedding model for retrieval and semantic search", "family": "bge", "license": "Gemma", "limit": { "context": 8192, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/BAAI/bge-multilingual-gemma2" ], "total_billion": 9.241713152 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE Multilingual Gemma2", "open_weights": true, "release_date": "2024-07-25", "schema": 2, "type": "embedding", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-multilingual-gemma2" } ] }, "baai/bge-reranker-v2-m3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lightweight multilingual cross-encoder that scores query-document relevance", "family": "bge", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/BAAI/bge-reranker-v2-m3" ], "total_billion": 0.567755777 } } }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "BGE Reranker V2 M3", "open_weights": true, "release_date": "2024-03-15", "schema": 2, "type": "reranking", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-reranker-v2-m3" } ] }, "baai/bge-small-en-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Small English text embedding model producing 384-dimensional vectors", "family": "bge", "license": "MIT", "limit": { "context": 512, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "BGE Small EN v1.5", "open_weights": true, "release_date": "2023-09-12", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/BAAI/bge-small-en-v1.5" } ] }, "baichuan/baichuan-m2-32b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open medical model trained for clinical reasoning and patient interaction with on, off, and auto thinking modes", "family": "baichuan", "license": "Apache-2.0", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/baichuan-inc/Baichuan-M2-32B" ], "total_billion": 32.763876352 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Baichuan M2 32B", "open_weights": true, "release_date": "2025-08-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/baichuan-inc/Baichuan-M2-32B" } ] }, "baidu/cobuddy": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Baidu Qianfan model optimized for code generation and tool-using coding agents", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "CoBuddy", "open_weights": false, "release_date": "2026-05-06", "schema": 2 }, "baidu/ernie-4.5-21b-a3b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open ERNIE text MoE with 21B total and 3B active parameters", "family": "ernie", "license": "Apache-2.0", "limit": { "context": 131072, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-PT" ], "total_billion": 21 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "ERNIE 4.5 21B A3B", "open_weights": true, "release_date": "2025-06-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-PT" } ] }, "baidu/ernie-4.5-300b-a47b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open ERNIE text MoE with 300B total and 47B active parameters", "family": "ernie", "license": "Apache-2.0", "limit": { "context": 131072, "output": 12000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 47, "sources": [ "https://huggingface.co/baidu/ERNIE-4.5-300B-A47B-Paddle" ], "total_billion": 300 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "ERNIE 4.5 300B A47B", "open_weights": true, "release_date": "2025-06-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/baidu/ERNIE-4.5-300B-A47B-Paddle" } ] }, "baidu/ernie-4.5-vl-424b-a47b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multimodal reasoning model for visual analysis, planning, and tool use", "family": "ernie", "knowledge": "2025-03-31", "limit": { "context": 123000, "output": 16000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 47, "sources": [ "https://huggingface.co/baidu/ERNIE-4.5-VL-424B-A47B-PT" ], "total_billion": 424 } } }, "modalities": { "input": [ "image/jpeg", "image/png", "image/bmp", "text" ], "output": [ "text" ] }, "name": "ERNIE 4.5 VL 424B A47B ", "open_weights": true, "release_date": "2025-06-30", "schema": 2 }, "baseten/elephant-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Base Labs' 9B open-weight decision model on Qwen3.5-9B that returns calibrated probabilities for typed choice, score and yes/no questions about text and images", "license": "MIT", "limit": { "context": 16384, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/baselabs/elephant-1" ], "total_billion": 9 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "elephant-1", "open_weights": true, "release_date": "2026-10-19", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/baselabs/elephant-1" } ] }, "baseten/mammoth-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Base Labs' 27B open-weight decision model on Qwen3.5-27B that returns calibrated probabilities for typed choice, score and yes/no questions about text and images", "license": "MIT", "limit": { "context": 16384, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/baselabs/mammoth-1" ], "total_billion": 27 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "mammoth-1", "open_weights": true, "release_date": "2026-10-19", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/baselabs/mammoth-1" } ] }, "black-forest-labs/flux-1-dev": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight FLUX.1 text-to-image model for high-quality guidance-distilled generation", "family": "flux", "license": "FLUX.1 [dev] Non-Commercial License", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.1-dev" ], "total_billion": 12 } ], "notes": "Transformer count only; a complete total including separately loaded encoders is not inferred." } } }, "modalities": { "input": [ "text" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.1 [dev]", "open_weights": true, "release_date": "2024-08-01", "schema": 2, "type": "image", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.1-dev" } ] }, "black-forest-labs/flux-1-kontext-dev": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight FLUX.1 Kontext model for instruction-based image editing", "family": "flux", "license": "FLUX.1 [dev] Non-Commercial License", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev" ], "total_billion": 12 } ], "notes": "Transformer count only; a complete total including separately loaded encoders is not inferred." } } }, "modalities": { "input": [ "text", "image" ], "output": [ "image" ] }, "name": "FLUX.1 Kontext [dev]", "open_weights": true, "release_date": "2025-05-29", "schema": 2, "type": "image", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev" } ] }, "black-forest-labs/flux-1-schnell": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast few-step FLUX.1 text-to-image model under Apache 2.0", "family": "flux", "license": "Apache 2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.1-schnell" ], "total_billion": 12 } ], "notes": "Transformer count only; a complete total including separately loaded encoders is not inferred." } } }, "modalities": { "input": [ "text" ], "output": [ "image" ] }, "name": "FLUX.1 [schnell]", "open_weights": true, "release_date": "2024-08-01", "schema": 2, "type": "image", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.1-schnell" } ] }, "black-forest-labs/flux-2-dev": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight 32B FLUX.2 image generation and multi-reference editing model", "family": "flux", "license": "FLUX Non-Commercial License", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.2-dev" ], "total_billion": 32 } ], "notes": "Transformer count only; a complete total including separately loaded encoders is not inferred." } } }, "modalities": { "input": [ "text", "image" ], "output": [ "image" ] }, "name": "FLUX.2 [dev]", "open_weights": true, "release_date": "2025-11-25", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.2-dev" } ] }, "black-forest-labs/flux-2-flex": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and editing model with adjustable steps and guidance, tuned for typography", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.2 [flex]", "open_weights": false, "release_date": "2025-11-25", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-2-klein-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast 4B open-weight image generation and editing model", "family": "flux", "license": "apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B" ], "total_billion": 4 } ], "notes": "Transformer count only; separate text encoder and autoencoder sizes are not added to invent a total." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.2 [klein] 4B", "open_weights": true, "release_date": "2026-01-15", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B" } ] }, "black-forest-labs/flux-2-klein-9b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast 9B open-weight image generation and multi-reference editing model", "family": "flux", "license": "FLUX Non-Commercial License", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Rectified flow transformer", "sources": [ "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B" ], "total_billion": 9 } ], "notes": "Transformer count only; separate text encoder and autoencoder sizes are not added to invent a total." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.2 [klein] 9B", "open_weights": true, "release_date": "2026-01-15", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B" } ] }, "black-forest-labs/flux-2-max": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Highest-quality FLUX.2 image model with grounding search and multi-reference editing", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.2 [max]", "open_weights": false, "release_date": "2025-12-16", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-2-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Production image generation and multi-reference editing model", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "FLUX.2 [pro]", "open_weights": false, "release_date": "2025-11-25", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-3-image": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and editing model with bounding-box layout control and up to ten references", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/png" ] }, "name": "FLUX 3 Image", "open_weights": false, "release_date": "2026-10-01", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-3-image-klein": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "5.6B open-weight FLUX 3 image generation and editing model", "family": "flux", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Image transformer", "notes": "Excludes the separately provided Qwen3-VL text encoder and image autoencoder.", "sources": [ "https://huggingface.co/black-forest-labs/flux-3-image-klein" ], "total_billion": 5.6 } ], "notes": "The official card explicitly limits its size label to the image transformer; complete total is unknown." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/jpeg", "image/png" ] }, "name": "FLUX 3 Image Klein", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/black-forest-labs/flux-3-image-klein" } ] }, "black-forest-labs/flux-3-video": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model with synchronized audio for text, keyframe and continuation workflows", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "FLUX 3 Video", "open_weights": false, "release_date": "2026-08-04", "schema": 2, "type": "video", "v2_only": true }, "black-forest-labs/flux-kontext-max": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Highest-quality FLUX.1 Kontext model for image generation and in-context editing with stronger prompt adherence and typography", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/jpeg", "image/png" ] }, "name": "FLUX.1 Kontext [max]", "open_weights": false, "release_date": "2025-05-29", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-kontext-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and in-context editing model that edits images from simple text instructions", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/jpeg", "image/png" ] }, "name": "FLUX.1 Kontext [pro]", "open_weights": false, "release_date": "2025-05-29", "schema": 2, "type": "image", "v2_only": true }, "black-forest-labs/flux-video-edit": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Prompt-based video editing model that adds, removes or replaces elements while keeping motion and timing", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "FLUX Video Edit [fast]", "open_weights": false, "release_date": "2026-09", "schema": 2, "type": "video", "v2_only": true }, "black-forest-labs/flux-video-upscale": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "FLUX 3 video super-resolution model that upscales clips 1.5x to 3x while keeping the audio track", "family": "flux", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "FLUX Video Upscale", "open_weights": false, "release_date": "2026-08-20", "schema": 2, "type": "video", "v2_only": true }, "bria/fibo-generate-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-image model driven by structured JSON prompts, trained on licensed data", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "DiT", "notes": "Official few-step release retains the base FIBO DiT architecture; not a full-pipeline parameter count.", "sources": [ "https://huggingface.co/briaai/Fibo-1.5" ], "total_billion": 8 } ], "notes": "Only the DiT architecture count is used; the text encoder and VAE are separate components with no complete total inferred." } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg" ] }, "name": "FIBO Generate 1.5", "open_weights": true, "release_date": "2026-08-24", "schema": 2, "type": "image", "v2_only": true }, "butler/cicero": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight decision model built on Qwen3.5-35B-A3B that answers typed yes/no, choice and score questions about text, chats or images with calibrated probabilities", "license": "Apache-2.0", "limit": { "context": 131072, "output": 0 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/butler-compute/Cicero" } ], "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "Cicero", "open_weights": true, "release_date": "2026-11-05", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "format": "safetensors", "label": "Hugging Face", "url": "https://huggingface.co/butler-compute/Cicero" }, { "format": "safetensors", "label": "Hugging Face (NVFP4)", "quantization": "nvfp4", "url": "https://huggingface.co/butler-compute/Cicero-NVFP4" } ] }, "bytedance-seed/seed-1-6": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "ByteDance Seed model for long-context reasoning, instruction following, and tool-assisted tasks", "family": "seed", "limit": { "context": 256000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 1.6", "open_weights": false, "release_date": "2025-10-15", "schema": 2 }, "bytedance-seed/seed-1-6-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Low-latency ByteDance Seed model for high-throughput chat, extraction, and lightweight tool use", "family": "seed", "limit": { "context": 256000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 1.6 Flash", "open_weights": false, "release_date": "2025-08-28", "schema": 2 }, "bytedance-seed/seed-1-6-vision": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "ByteDance Seed multimodal model for image understanding, visual reasoning, and tool-assisted tasks", "family": "seed", "limit": { "context": 256000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic" ], "output": [ "text" ] }, "name": "Seed 1.6 Vision", "open_weights": false, "release_date": "2025-08-15", "schema": 2 }, "bytedance-seed/seed-1-8": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "ByteDance Seed model for multimodal reasoning, long-context analysis, and agent workflows", "family": "seed", "limit": { "context": 256000, "output": 64000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic" ], "output": [ "text" ] }, "name": "Seed 1.8", "open_weights": false, "release_date": "2025-12-28", "schema": 2 }, "bytedance-seed/seed-2.0-code": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "ByteDance Seed coding model for multimodal software engineering and long-running agents", "family": "seed", "limit": { "context": 262144, "output": 131072 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.0 Code", "open_weights": false, "release_date": "2026-02-14", "schema": 2 }, "bytedance-seed/seed-2.0-lite": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Cost-efficient ByteDance Seed 2.0 model for production chat, analysis, and structured generation", "family": "seed", "limit": { "context": 256000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.0 Lite", "open_weights": false, "release_date": "2026-02-14", "schema": 2 }, "bytedance-seed/seed-2.0-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Lightweight ByteDance Seed 2.0 model for low-latency multimodal reasoning and high-volume tasks", "family": "seed", "limit": { "context": 256000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.0 Mini", "open_weights": false, "release_date": "2026-02-14", "schema": 2 }, "bytedance-seed/seed-2.0-pro": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Flagship ByteDance Seed 2.0 model for complex multimodal reasoning and long-horizon agent workflows", "family": "seed", "limit": { "context": 256000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.0 Pro", "open_weights": false, "release_date": "2026-02-14", "schema": 2 }, "bytedance-seed/seed-2.1-lite": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Omnimodal ByteDance Seed 2.1 model with native text, vision, and audio understanding, 1M context, and improved coding and agent task completion", "family": "seed", "limit": { "context": 1024000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "audio/mpeg", "audio/wav", "audio/aac", "audio/x-m4a", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.1 Lite", "open_weights": false, "release_date": "2026-09-15", "schema": 2 }, "bytedance-seed/seed-2.1-pro": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Flagship ByteDance Seed 2.1 model for complex multimodal reasoning, coding, and agents", "family": "seed", "limit": { "context": 1024000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.1 Pro", "open_weights": false, "release_date": "2026-06-23", "schema": 2 }, "bytedance-seed/seed-2.1-turbo": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Faster ByteDance Seed 2.1 model for multimodal reasoning and latency-sensitive agent workflows", "family": "seed", "limit": { "context": 256000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed 2.1 Turbo", "open_weights": false, "release_date": "2026-06-23", "schema": 2 }, "bytedance-seed/seed-audio-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Audio creation model generating speech, sound effects, and ambience from text prompts", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Seed Audio 1.0", "open_weights": false, "release_date": "2026-07-20", "schema": 2, "type": "speech", "v2_only": true }, "bytedance-seed/seed-character": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "ByteDance Seed model optimized for character-driven dialogue and consistent conversational behavior", "family": "seed", "limit": { "context": 256000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed Character", "open_weights": false, "release_date": "2026-06-23", "schema": 2 }, "bytedance-seed/seed-evolving": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Rolling ByteDance Seed model for rapidly updated reasoning, coding, and agent capabilities", "family": "seed", "limit": { "context": 1024000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/x-icon", "image/x-icns", "image/sgi", "image/jp2", "image/heic", "video/mp4", "video/x-msvideo", "video/quicktime" ], "output": [ "text" ] }, "name": "Seed Evolving", "open_weights": false, "release_date": "2026-06-23", "schema": 2 }, "bytedance-seed/seed-oss-36b-instruct": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open ByteDance Seed model for long-context reasoning, tool use, and agents", "family": "seed", "knowledge": "2024-07", "license": "Apache 2.0", "limit": { "context": 524288, "output": 524288 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct" ], "total_billion": 36 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Seed-OSS-36B-Instruct", "open_weights": true, "release_date": "2025-08-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct" } ] }, "bytedance-seed/seedance-1.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Seedance 1.0 large-parameter video model with multi-shot storytelling, from text, a first frame or first and last frames", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif" ], "output": [ "video/mp4" ] }, "name": "Seedance 1.0 Pro", "open_weights": false, "release_date": "2025-06-11", "schema": 2, "type": "video", "v2_only": true }, "bytedance-seed/seedance-1.0-pro-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost Seedance 1.0 Pro variant for text-to-video and first-frame image-to-video", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif" ], "output": [ "video/mp4" ] }, "name": "Seedance 1.0 Pro Fast", "open_weights": false, "release_date": "2025-10-24", "schema": 2, "type": "video", "v2_only": true }, "bytedance-seed/seedance-1.5-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Audio-visual model that generates video and sound together from text or images", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif" ], "output": [ "video/mp4" ] }, "name": "Seedance 1.5 Pro", "open_weights": false, "release_date": "2025-12-16", "schema": 2, "type": "video", "v2_only": true }, "bytedance-seed/seedance-2.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "ByteDance unified multimodal audio-video model supporting text, image, video and audio references, editing, extension and synchronized stereo sound.", "family": "seed", "last_updated": "2026-10-04", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://seed.bytedance.com/en/blog/seedance-2-0-official-launch" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4", "audio" ] }, "name": "Seedance 2.0", "open_weights": false, "release_date": "2026-02-12", "schema": 2, "type": "video" }, "bytedance-seed/seedance-2.0-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost Seedance 2.0 variant for text, image, video and audio references with synchronized sound.", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4", "audio" ] }, "name": "Seedance 2.0 Fast", "open_weights": false, "release_date": "2026-02-12", "schema": 2, "type": "video", "v2_only": true }, "bytedance-seed/seedance-2.0-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lower-cost, faster Seedance 2.0 audio-video model at 480p and 720p", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Seedance 2.0 Mini", "open_weights": false, "release_date": "2026-06-15", "schema": 2, "type": "video", "v2_only": true }, "bytedance-seed/seedance-2.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "ByteDance multimodal audio-video model with up to 30-second single-pass generation, multi-round extension, reference inputs and timestamp-level editing.", "family": "seed", "last_updated": "2026-10-04", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://seed.bytedance.com/en/blog/one-take-creation-flexible-referencing-introducing-seedance-2-5" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4", "audio" ] }, "name": "Seedance 2.5", "open_weights": false, "release_date": "2026-07-31", "schema": 2, "type": "video" }, "bytedance-seed/seedream-4.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Unified image generation and editing model with multi-image references", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif" ], "output": [ "image/jpeg" ] }, "name": "Seedream 4.5", "open_weights": false, "release_date": "2025-12-03", "schema": 2, "type": "image", "v2_only": true }, "bytedance-seed/seedream-5.0-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast, low-cost Seedream 5.0 image generation and editing model", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Seedream 5.0 Flash", "open_weights": false, "release_date": "2026-09", "schema": 2, "type": "image", "v2_only": true }, "bytedance-seed/seedream-5.0-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation model with deeper reasoning and real-time web search", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Seedream 5.0 Lite", "open_weights": false, "release_date": "2026-02-13", "schema": 2, "type": "image", "v2_only": true }, "bytedance-seed/seedream-5.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and interactive editing model for commercial visual production", "family": "seed", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/bmp", "image/tiff", "image/gif", "image/heic", "image/heif" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Seedream 5.0 Pro", "open_weights": false, "release_date": "2026-07-08", "schema": 2, "type": "image", "v2_only": true }, "bytedance-seed/stable-diffusion-xl-lightning": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Few-step distilled Stable Diffusion XL model for fast 1024px text-to-image", "family": "stable-diffusion", "license": "CreativeML Open RAIL++-M", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "1-step U-Net (x0)", "notes": "Exact stored F16 weight count verified from this file's tensor header; no model weight data downloaded.", "sources": [ "https://huggingface.co/ByteDance/SDXL-Lightning/blob/c9a24f48e1c025556787b0c58dd67a091ece2e44/sdxl_lightning_1step_unet_x0.safetensors", "https://huggingface.co/ByteDance/SDXL-Lightning" ], "total_billion": 2.567463684 }, { "name": "2-step U-Net", "notes": "Exact stored F16 weight count verified from this alternative checkpoint's tensor header.", "sources": [ "https://huggingface.co/ByteDance/SDXL-Lightning/blob/c9a24f48e1c025556787b0c58dd67a091ece2e44/sdxl_lightning_2step_unet.safetensors" ], "total_billion": 2.567463684 }, { "name": "4-step U-Net", "notes": "Exact stored F16 weight count verified from this alternative checkpoint's tensor header.", "sources": [ "https://huggingface.co/ByteDance/SDXL-Lightning/blob/c9a24f48e1c025556787b0c58dd67a091ece2e44/sdxl_lightning_4step_unet.safetensors" ], "total_billion": 2.567463684 }, { "name": "8-step U-Net", "notes": "Exact stored F16 weight count verified from this alternative checkpoint's tensor header.", "sources": [ "https://huggingface.co/ByteDance/SDXL-Lightning/blob/c9a24f48e1c025556787b0c58dd67a091ece2e44/sdxl_lightning_8step_unet.safetensors" ], "total_billion": 2.567463684 } ], "notes": "Alternative step-specific U-Net checkpoints, not additive modules. No default step count or complete SDXL pipeline total is selected." } } }, "modalities": { "input": [ "text" ], "output": [ "image" ] }, "name": "SDXL-Lightning", "open_weights": true, "release_date": "2024-02-21", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ByteDance/SDXL-Lightning" } ] }, "canopylabs/orpheus-3b-0.1-ft": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open English text-to-speech model fine-tuned for natural prosody and expressive delivery", "family": "canopylabs", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the complete TTS checkpoint, not the nominal 3B designation.", "sources": [ "https://huggingface.co/api/models/canopylabs/orpheus-3b-0.1-ft" ], "total_billion": 3.782986752 } } }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Orpheus 3B", "open_weights": true, "release_date": "2025-03", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/canopylabs/orpheus-3b-0.1-ft" } ] }, "canopylabs/orpheus-arabic-saudi": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive Saudi Arabic text-to-speech model with bracketed vocal directions", "family": "canopylabs", "limit": { "context": 4000, "output": 50000 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav" ] }, "name": "Orpheus Arabic Saudi", "open_weights": false, "release_date": "2025-12-16", "schema": 2, "type": "speech" }, "canopylabs/orpheus-v1-english": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive English text-to-speech model with bracketed vocal directions", "family": "canopylabs", "limit": { "context": 4000, "output": 50000 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav" ] }, "name": "Orpheus V1 English", "open_weights": false, "release_date": "2025-12-19", "schema": 2, "type": "speech" }, "cloudflare/clef": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Cloudflare's open-weight decision model that returns probabilities for typed choice, score and yes/no questions about text or images", "license": "Apache-2.0", "limit": { "context": 65536, "output": 0 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://blog.cloudflare.com/clef-decision-models/" }, { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/Cloudflare/clef" } ], "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Backbone (including vision encoder)", "notes": "Hub BF16 count independently verified against all twelve backbone shard headers; excludes the separate joint head.", "sources": [ "https://huggingface.co/Cloudflare/clef/blob/cfddda043dd190ef3b01cff449d5ee8cff1e8ec6/model.safetensors.index.json", "https://huggingface.co/api/models/Cloudflare/clef" ], "total_billion": 27.35672856 }, { "name": "Joint schema head", "notes": "Stored BF16 parameter count verified from the head's tensor header.", "sources": [ "https://huggingface.co/Cloudflare/clef/blob/cfddda043dd190ef3b01cff449d5ee8cff1e8ec6/joint_head.safetensors" ], "total_billion": 0.128056324 } ], "notes": "Backbone and joint head are recorded separately; no complete model total is generated by summing them." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "text" ] }, "name": "Clef", "open_weights": true, "release_date": "2026-10-01", "schema": 2, "type": "decision", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Cloudflare/clef" } ] }, "cloudflare/clef-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Cloudflare's fast 9B open-weight decision model that returns probabilities for typed choice, score and yes/no questions about text or images", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://blog.cloudflare.com/clef-decision-models/" }, { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/Cloudflare/clef-flash" } ], "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Backbone (including vision encoder)", "notes": "Hub BF16 count independently verified against all four backbone shard headers; excludes the separate joint head.", "sources": [ "https://huggingface.co/Cloudflare/clef-flash/blob/fb4417d31c1d0f9e7e77f1141b40aaede7491e00/model.safetensors.index.json", "https://huggingface.co/api/models/Cloudflare/clef-flash" ], "total_billion": 9.409813744 }, { "name": "Joint schema head", "notes": "Stored BF16 parameter count verified from the head's tensor header.", "sources": [ "https://huggingface.co/Cloudflare/clef-flash/blob/fb4417d31c1d0f9e7e77f1141b40aaede7491e00/joint_head.safetensors" ], "total_billion": 0.12176282 } ], "notes": "Backbone and joint head are recorded separately; no complete model total is generated by summing them." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "text" ] }, "name": "Clef Flash", "open_weights": true, "release_date": "2026-10-01", "schema": 2, "type": "decision", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Cloudflare/clef-flash" } ] }, "cognitivecomputations/dolphin-mistral-24b-venice-edition": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral", "knowledge": "2024-04-30", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/cognitivecomputations/Dolphin-Mistral-24B-Venice-Edition" ], "total_billion": 24.01136128 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Uncensored", "open_weights": true, "release_date": "2025-07-09", "schema": 2 }, "cohere/c4ai-aya-expanse-32b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open multilingual model optimized for generation across 23 languages", "license": "CC-BY-NC-4.0", "limit": { "context": 128000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/aya-expanse-32b" ], "total_billion": 32 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aya Expanse 32B", "open_weights": true, "release_date": "2024-10-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/aya-expanse-32b" } ] }, "cohere/c4ai-aya-expanse-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Compact open multilingual model optimized for generation across 23 languages", "license": "CC-BY-NC-4.0", "limit": { "context": 8000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/aya-expanse-8b" ], "total_billion": 8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Aya Expanse 8B", "open_weights": true, "release_date": "2024-10-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/aya-expanse-8b" } ] }, "cohere/c4ai-aya-vision-32b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open multilingual vision model for OCR, visual reasoning, and image question answering", "last_updated": "2025-05-14", "license": "CC-BY-NC-4.0", "limit": { "context": 16000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 count for the complete multimodal checkpoint, not the card's rounded 32B size designation.", "sources": [ "https://huggingface.co/api/models/CohereLabs/aya-vision-32b" ], "total_billion": 33.137896688 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Aya Vision 32B", "open_weights": true, "release_date": "2025-03-04", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/aya-vision-32b" } ] }, "cohere/c4ai-aya-vision-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Compact open multilingual vision model for OCR and visual question answering", "last_updated": "2025-05-14", "license": "CC-BY-NC-4.0", "limit": { "context": 16000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 count for the complete multimodal checkpoint, not the card's rounded 8B size designation.", "sources": [ "https://huggingface.co/api/models/CohereLabs/aya-vision-8b" ], "total_billion": 8.631842032 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Aya Vision 8B", "open_weights": true, "release_date": "2025-03-04", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/aya-vision-8b" } ] }, "cohere/cohere-transcribe-03-2026": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open 2B speech recognition model for accurate transcription across 14 languages", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/cohere-transcribe-03-2026" ], "total_billion": 2 } } }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/ogg", "audio/wav" ], "output": [ "text" ] }, "name": "Cohere Transcribe", "open_weights": true, "release_date": "2026-03-26", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/cohere-transcribe-03-2026" } ] }, "cohere/cohere-transcribe-arabic-07-2026": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Cohere Transcribe fine-tune for Arabic dialects and Arabic-English code-switching", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/cohere-transcribe-arabic-07-2026" ], "total_billion": 2 } } }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/ogg", "audio/wav" ], "output": [ "text" ] }, "name": "Cohere Transcribe Arabic", "open_weights": true, "release_date": "2026-07-07", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/cohere-transcribe-arabic-07-2026" } ] }, "cohere/command-a": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere command model for multilingual enterprise agents, tools, and chat", "family": "command-a", "knowledge": "2024-08-31", "limit": { "context": 256000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "The March 2025 Command A release represented by this local entry, not the later Command A+ model.", "sources": [ "https://huggingface.co/CohereLabs/c4ai-command-a-03-2025" ], "total_billion": 111 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command A", "open_weights": true, "release_date": "2025-03-13", "schema": 2 }, "cohere/command-a-03-2025": { "benchmarks": [ { "date": "2025-03-14", "metric": "percent correct", "name": "Aider Polyglot", "score": 12.0, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere command model for multilingual enterprise agents, tools, and chat", "family": "command-a", "knowledge": "2024-06-01", "limit": { "context": 256000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/c4ai-command-a-03-2025" ], "total_billion": 111 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command A", "open_weights": true, "release_date": "2025-03-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-a-03-2025" } ] }, "cohere/command-a-plus-05-2026": { "benchmarks": [ { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-command-a-plus", "variant": "pass@1 averaged over 3 repeats", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-command-a-plus", "variant": "pass@1 averaged over 3 repeats", "version": "0.1.0" }, { "dataset": "May 2025 revision; 2,158 text-only questions", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 12, "source": "https://artificialanalysis.ai/models/comparisons/command-a-plus-vs-claude-sonnet-4-6-adaptive", "variant": "pass@1" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 6.24120025576958, "source": "https://artificialanalysis.ai/models/command-a-plus", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere's stronger command model for multilingual agents and enterprise workflows", "family": "command-a", "knowledge": "2025-04-01", "last_updated": "2026-06-09", "limit": { "context": 128000, "output": 64000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 25, "sources": [ "https://huggingface.co/CohereLabs/command-a-plus-05-2026-bf16" ], "total_billion": 218 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Command A Plus", "open_weights": true, "release_date": "2026-05-20", "schema": 2 }, "cohere/command-a-reasoning-08-2025": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere reasoning model for multilingual enterprise agents, tools, and complex workflows", "family": "command-a", "knowledge": "2024-06-01", "limit": { "context": 256000, "output": 32000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/command-a-reasoning-08-2025" ], "total_billion": 111 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command A Reasoning", "open_weights": true, "release_date": "2025-08-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-a-reasoning-08-2025" } ] }, "cohere/command-a-translate-08-2025": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Translation model for multilingual conversion, localization, and cross-language workflows", "family": "command-a", "knowledge": "2024-06-01", "limit": { "context": 8000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/command-a-translate-08-2025" ], "total_billion": 111 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command A Translate", "open_weights": true, "release_date": "2025-08-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-a-translate-08-2025" } ] }, "cohere/command-a-vision-07-2025": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Cohere vision model for multilingual document analysis, OCR, and image understanding", "family": "command-a", "knowledge": "2024-06-01", "limit": { "context": 128000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/command-a-vision-07-2025" ], "total_billion": 112 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Command A Vision", "open_weights": true, "release_date": "2025-07-31", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-a-vision-07-2025" } ] }, "cohere/command-r-08-2024": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere retrieval model for long-context chat and enterprise RAG workflows", "family": "command-r", "knowledge": "2024-06-01", "limit": { "context": 128000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/c4ai-command-r-08-2024" ], "total_billion": 32 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command R", "open_weights": true, "release_date": "2024-08-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-r-08-2024" } ] }, "cohere/command-r-plus-08-2024": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere's RAG workhorse for long-context enterprise search and tool use", "family": "command-r", "knowledge": "2024-06-01", "limit": { "context": 128000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/c4ai-command-r-plus-08-2024" ], "total_billion": 104 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command R+", "open_weights": true, "release_date": "2024-08-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-r-plus-08-2024" } ] }, "cohere/command-r7b-12-2024": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere retrieval model for long-context chat and enterprise RAG workflows", "family": "command-r", "knowledge": "2024-06-01", "limit": { "context": 128000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint including embeddings, not the nominal 7B designation.", "sources": [ "https://huggingface.co/api/models/CohereLabs/c4ai-command-r7b-12-2024" ], "total_billion": 8.028033024 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command R7B", "open_weights": true, "release_date": "2024-12-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-r7b-12-2024" } ] }, "cohere/command-r7b-arabic-02-2025": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Command R model optimized for Arabic enterprise chat, RAG, and cultural knowledge", "family": "command-r", "knowledge": "2024-06-01", "limit": { "context": 128000, "output": 4000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Approximately 8B including embeddings, as explicitly reported by the card; the 7B designation covers transformer parameters only.", "sources": [ "https://huggingface.co/CohereLabs/c4ai-command-r7b-arabic-02-2025" ], "total_billion": 8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Command R7B Arabic", "open_weights": true, "release_date": "2025-02-27", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/c4ai-command-r7b-arabic-02-2025" } ] }, "cohere/embed-english-light-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Smaller, faster English embedding model", "family": "cohere-embed", "limit": { "context": 512, "output": 384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed English Light 3", "open_weights": false, "release_date": "2023-11-02", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-english-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English embedding model for search, classification, and clustering", "family": "cohere-embed", "limit": { "context": 512, "output": 1024 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed English 3", "open_weights": false, "release_date": "2023-11-02", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-multilingual-light-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Smaller, faster multilingual embedding model", "family": "cohere-embed", "limit": { "context": 512, "output": 384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed Multilingual Light 3", "open_weights": false, "release_date": "2023-11-02", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-multilingual-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual embedding model covering over 100 languages", "family": "cohere-embed", "limit": { "context": 512, "output": 1024 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed Multilingual 3", "open_weights": false, "release_date": "2023-11-02", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-v4.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multimodal embedding model for text, images, and mixed text-and-image documents", "family": "cohere-embed", "limit": { "context": 128000, "output": 1536 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed 4", "open_weights": false, "release_date": "2025-04-15", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-v5.0-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lighter Embed 5 model sharing Embed 5 Pro's embedding space", "family": "cohere-embed", "limit": { "context": 128000, "output": 2048 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed 5 Fast", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/embed-v5.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Cohere's most capable multimodal embedding model for text, images, and mixed documents", "family": "cohere-embed", "limit": { "context": 128000, "output": 2048 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "embedding" ] }, "name": "Embed 5 Pro", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "embedding", "v2_only": true }, "cohere/north-mini-code-1-0": { "benchmarks": [ { "date": "2026-06-09", "harness": "SWE-agent", "metric": "resolved", "name": "SWE-Bench Verified", "score": 67.6, "source": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0" }, { "date": "2026-06-09", "harness": "SWE-agent", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 40.2, "source": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0" }, { "date": "2026-06-09", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 27.6, "source": "https://artificialanalysis.ai/articles/north-mini-code-cohere-s-small-coding-focused-moe-model" }, { "date": "2026-06-09", "metric": "index score", "name": "Artificial Analysis Coding Index", "score": 33.4, "source": "https://artificialanalysis.ai/articles/north-mini-code-cohere-s-small-coding-focused-moe-model" }, { "date": "2026-06-09", "metric": "win rate", "name": "GDPval-AA", "score": 14, "source": "https://artificialanalysis.ai/articles/north-mini-code-cohere-s-small-coding-focused-moe-model" }, { "date": "2026-06-09", "metric": "success rate", "name": "τ²-Bench Telecom", "score": 37, "source": "https://artificialanalysis.ai/articles/north-mini-code-cohere-s-small-coding-focused-moe-model" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 3.42470262148493, "source": "https://artificialanalysis.ai/models/north-mini-code", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Cohere coding model for practical software engineering and agentic edits", "family": "north", "knowledge": "2025-09-23", "limit": { "context": 256000, "output": 64000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/CohereLabs/North-Mini-Code-1.0" ], "total_billion": 30 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "North Mini Code", "open_weights": true, "release_date": "2026-06-09", "schema": 2 }, "cohere/north-small-translate-1-0": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight translation model for more than 50 languages and locale variants", "family": "north", "license": "CC-BY-NC-4.0", "limit": { "context": 16000, "output": 16000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 25, "sources": [ "https://huggingface.co/CohereLabs/North-Small-Translate-1.0" ], "total_billion": 218 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "North Small Translate", "open_weights": true, "release_date": "2026-09-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/North-Small-Translate-1.0" } ] }, "cohere/rerank-english-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English reranker for documents and semi-structured data", "limit": { "context": 4096, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank English 3", "open_weights": false, "release_date": "2024-04-11", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-multilingual-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Reranker for non-English documents and semi-structured data", "limit": { "context": 4096, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank Multilingual 3", "open_weights": false, "release_date": "2024-04-11", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-v3.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual reranker for documents and semi-structured data", "limit": { "context": 4096, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank 3.5", "open_weights": false, "release_date": "2024-12-02", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-v4.0-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency multilingual reranker for high-throughput retrieval", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank 4 Fast", "open_weights": false, "release_date": "2025-12-11", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-v4.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual reranker for maximum relevance on documents and semi-structured data", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank 4 Pro", "open_weights": false, "release_date": "2025-12-11", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-v5.0-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency multilingual reranker for high-volume search and agent loops", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank 5 Fast", "open_weights": false, "release_date": "2026-10-21", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/rerank-v5.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Highest-quality multilingual reranker for reasoning-heavy queries and long documents", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "Rerank 5 Pro", "open_weights": false, "release_date": "2026-10-21", "schema": 2, "type": "reranking", "v2_only": true }, "cohere/tiny-aya-earth": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multilingual model specialized for West Asian and African languages", "license": "CC-BY-NC-4.0", "limit": { "context": 8000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/tiny-aya-earth" ], "total_billion": 3.35 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Tiny Aya Earth", "open_weights": true, "release_date": "2026-02-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/tiny-aya-earth" } ] }, "cohere/tiny-aya-fire": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multilingual model specialized for South Asian languages", "license": "CC-BY-NC-4.0", "limit": { "context": 8000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/tiny-aya-fire" ], "total_billion": 3.35 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Tiny Aya Fire", "open_weights": true, "release_date": "2026-02-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/tiny-aya-fire" } ] }, "cohere/tiny-aya-global": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multilingual model balanced across 70 languages and regions", "license": "CC-BY-NC-4.0", "limit": { "context": 8000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/tiny-aya-global" ], "total_billion": 3.35 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Tiny Aya Global", "open_weights": true, "release_date": "2026-02-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/tiny-aya-global" } ] }, "cohere/tiny-aya-water": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multilingual model specialized for European and Asia-Pacific languages", "license": "CC-BY-NC-4.0", "limit": { "context": 8000, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/CohereLabs/tiny-aya-water" ], "total_billion": 3.35 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Tiny Aya Water", "open_weights": true, "release_date": "2026-02-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/CohereLabs/tiny-aya-water" } ] }, "deepgram/aura-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Enterprise text-to-speech model for real-time voice agents", "family": "aura", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/pcm", "audio/basic", "audio/alaw", "audio/mpeg", "audio/opus", "audio/flac", "audio/aac" ] }, "name": "Aura-2", "open_weights": false, "release_date": "2025-04-15", "schema": 2, "type": "speech", "v2_only": true }, "deepgram/flux-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Conversation-native text-to-speech model for real-time voice agents", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Flux TTS", "open_weights": false, "release_date": "2026-08-12", "schema": 2, "type": "speech", "v2_only": true }, "deepgram/nova-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose speech-to-text model with monolingual and multilingual transcription", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/mpeg", "audio/mp4", "audio/aac", "audio/wav", "audio/flac", "audio/ogg", "audio/opus", "audio/webm" ], "output": [ "text" ] }, "name": "Nova-3", "open_weights": false, "release_date": "2025-02-12", "schema": 2, "type": "transcription", "v2_only": true }, "deepreinforce/ornith-1.0-31b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open coding-reasoning model for repository tasks and self-improving agents", "family": "ornith", "license": "MIT", "limit": { "context": 262144 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://deep-reinforce.com/ornith_1_0.html" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Author-reported rounded architecture size, not an independently counted checkpoint: the original June 25, 2026 Ornith-1.0 README explicitly lists the available model scales and architectures as 9B-Dense, 31B-Dense, 35B-MoE and 397B-MoE, also stated in the official launch post. This records the author's 31B dense-model specification, not a number parsed from the local model ID or inherited from Gemma/Qwen. No accessible 31B checkpoint or exact serialized total has been verified; the repository's serving table lists only 9B, 35B and 397B releases.", "sources": [ "https://github.com/ornith-ai/Ornith-1/blob/d196d60faa9ee1b1daa811e8b552b4ee203b1272/README.md", "https://ornith.ai/ornith_1_0.html" ], "total_billion": 31 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.0 31B", "open_weights": true, "release_date": "2026-06-25", "schema": 2 }, "deepreinforce/ornith-1.0-35b": { "benchmarks": [ { "metric": "percent resolved", "name": "SWE-Bench Verified", "score": 75.6, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" }, { "metric": "percent resolved", "name": "SWE-Bench Pro", "score": 50.4, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" }, { "metric": "percent resolved", "name": "SWE-Bench Multilingual", "score": 69.3, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 64.2, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B", "variant": "Terminus-2" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 62.8, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B", "variant": "Claude Code" }, { "metric": "percent", "name": "NL2Repo", "score": 34.6, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" }, { "metric": "percent", "name": "Claw-eval", "score": 69.8, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Large coding-reasoning model for agentic software tasks and RL search", "family": "ornith", "license": "MIT", "limit": { "context": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" }, { "label": "Announcement", "type": "announcement", "url": "https://deep-reinforce.com/ornith_1_0.html" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Official announcement explicitly states 35B parameters. Uses the author's rounded specification, not the inconsistent Hub API total/dtype statistics.", "sources": [ "https://deep-reinforce.com/ornith_1_0.html" ], "total_billion": 35 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.0 35B", "open_weights": true, "release_date": "2026-06-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B" } ] }, "deepreinforce/ornith-1.0-397b": { "benchmarks": [ { "metric": "percent resolved", "name": "SWE-Bench Verified", "score": 82.4, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "metric": "percent resolved", "name": "SWE-Bench Pro", "score": 62.2, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "metric": "percent resolved", "name": "SWE-Bench Multilingual", "score": 78.9, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 77.5, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B", "variant": "Terminus-2" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 78.2, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B", "variant": "Claude Code" }, { "metric": "percent", "name": "NL2Repo", "score": 48.2, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "metric": "percent", "name": "Claw-eval", "score": 77.1, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Large coding-reasoning model for agentic software tasks and RL search", "family": "ornith", "license": "MIT", "limit": { "context": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "label": "Announcement", "type": "announcement", "url": "https://deep-reinforce.com/ornith_1_0.html" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint. Active count not inferred from the base model.", "sources": [ "https://huggingface.co/api/models/deepreinforce-ai/Ornith-1.0-397B" ], "total_billion": 396.802360816 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.0 397B", "open_weights": true, "release_date": "2026-06-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B" }, { "label": "Hugging Face (FP8)", "quantization": "fp8", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-397B-FP8" } ] }, "deepreinforce/ornith-1.0-9b": { "benchmarks": [ { "metric": "percent resolved", "name": "SWE-Bench Verified", "score": 69.4, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" }, { "metric": "percent resolved", "name": "SWE-Bench Pro", "score": 42.9, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" }, { "metric": "percent resolved", "name": "SWE-Bench Multilingual", "score": 52, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 43.1, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B", "variant": "Terminus-2" }, { "metric": "percent", "name": "Terminal-Bench 2.1", "score": 40.6, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B", "variant": "Claude Code" }, { "metric": "percent", "name": "NL2Repo", "score": 27.2, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" }, { "metric": "percent", "name": "Claw-eval", "score": 63.1, "source": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open coding-reasoning model for repository tasks and self-improving agents", "family": "ornith", "license": "MIT", "limit": { "context": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" }, { "label": "Announcement", "type": "announcement", "url": "https://deep-reinforce.com/ornith_1_0.html" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Official announcement explicitly describes a 9B-parameter model. Uses the author's rounded specification, not the inconsistent Hub API total/dtype statistics.", "sources": [ "https://deep-reinforce.com/ornith_1_0.html" ], "total_billion": 9 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.0 9B", "open_weights": true, "release_date": "2026-06-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B" } ] }, "deepreinforce/ornith-1.5-122b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mixture-of-experts coding-reasoning model for agentic software tasks, tool use, and image understanding", "family": "ornith", "license": "MIT", "limit": { "context": 262144, "output": 32768 }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.5 122B", "open_weights": true, "release_date": "2026-10-20", "schema": 2 }, "deepreinforce/ornith-1.5-35b-a3b": { "benchmarks": [ { "harness": "Claude Code", "name": "DeepSWE", "score": 22, "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B", "variant": "mean of 5 runs; temperature=1.0; top_p=0.95; context=256K" }, { "harness": "OpenHands", "name": "SWE-bench Pro", "score": 59.6, "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B", "variant": "mean of 5 runs; temperature=1.0; top_p=0.95; context=256K; network disabled; git history removed" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 25.6, "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B", "variant": "no tools; mean of 5 runs; Claude 4.6 Opus judge" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 33.4, "source": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B", "variant": "with tools; mean of 5 runs; Claude 4.6 Opus judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mixture-of-experts coding-reasoning model for agentic software tasks, tool use, and image understanding", "family": "ornith", "last_updated": "2026-08-23", "license": "MIT", "limit": { "context": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "notes": "Both counts are approximate in the model card's deployment section.", "sources": [ "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B" ], "total_billion": 35 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Ornith 1.5 35B A3B", "open_weights": true, "release_date": "2026-08-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B" } ] }, "deepseek/deepseek-chat": { "benchmarks": [ { "date": "2025-10-03", "metric": "percent correct", "name": "Aider Polyglot", "score": 70.2, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek chat model for instruction following, coding, and analysis", "family": "deepseek", "knowledge": "2025-09", "last_updated": "2026-02-28", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "notes": "Historical scope: the archived official February 2026 pricing table explicitly maps deepseek-chat to DeepSeek-V3.2 (Non-thinking Mode), matching this local December 2025 release and linked checkpoint. Counts are Artificial Analysis's V3.2 architecture specification; not a claim about the current moving API alias or attribution of older benchmark runs.", "sources": [ "https://web.archive.org/web/20260204135805id_/https://api-docs.deepseek.com/quick_start/pricing/", "https://api-docs.deepseek.com/news/news251201", "https://artificialanalysis.ai/models/deepseek-v3-2" ], "total_billion": 685 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek Chat", "open_weights": true, "release_date": "2025-12-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2" } ] }, "deepseek/deepseek-chat-v3-0324": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek chat model for instruction following, coding, and analysis", "family": "deepseek", "knowledge": "2024-07-31", "limit": { "context": 163840, "output": 147456 }, "metadata": { "modelatlas": { "parameters": { "notes": "OpenRouter's explicitly published size for deepseek/deepseek-chat-v3-0324, linked to DeepSeek-V3-0324. Its component-inclusion convention is not specified; no active count or 671B base-model total is substituted.", "sources": [ "https://openrouter.ai/api/v1/models", "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324" ], "total_billion": 685 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V3 0324", "open_weights": true, "release_date": "2025-03-24", "schema": 2 }, "deepseek/deepseek-chat-v3.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek chat model for instruction following, coding, and analysis", "family": "deepseek", "knowledge": "2025-03-31", "limit": { "context": 163840, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "notes": "The version-specific August 2025 V3.1 entry, not the moving deepseek-chat alias.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V3.1" ], "total_billion": 671 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V3.1", "open_weights": true, "release_date": "2025-08-21", "schema": 2 }, "deepseek/deepseek-ocr-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "High-accuracy OCR model for extracting text from documents, screenshots, receipts, and natural scenes", "limit": { "context": 8192, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete OCR checkpoint, including visual components.", "sources": [ "https://huggingface.co/api/models/deepseek-ai/DeepSeek-OCR-2" ], "total_billion": 3.38911936 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "DeepSeek OCR 2", "open_weights": true, "release_date": "2026-01-27", "schema": 2 }, "deepseek/deepseek-r1": { "benchmarks": [ { "date": "2025-01-20", "metric": "percent correct", "name": "Aider Polyglot", "score": 56.9, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 15.9, "source": "https://openrouter.ai/deepseek/deepseek-r1/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 35.7, "source": "https://openrouter.ai/deepseek/deepseek-r1/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 6.1, "source": "https://openrouter.ai/deepseek/deepseek-r1/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Classic open reasoning model for transparent math, coding, and deliberate problem solving", "family": "deepseek-thinking", "knowledge": "2024-07", "last_updated": "2025-05-29", "limit": { "context": 128000, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-R1" ], "total_billion": 671 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek-R1", "open_weights": true, "release_date": "2025-01-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" } ] }, "deepseek/deepseek-r1-0528": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "May 2025 DeepSeek R1 update with improved mathematics, coding, and reasoning", "family": "deepseek-thinking", "license": "MIT", "limit": { "context": 163840, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "notes": "Explicit size and active count in OpenRouter's deepseek/deepseek-r1-0528 description, linked there to this exact checkpoint. Retains its reported architecture convention rather than inferring from FP8 storage.", "sources": [ "https://openrouter.ai/api/v1/models", "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" ], "total_billion": 671 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek R1 0528", "open_weights": true, "release_date": "2025-05-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" } ] }, "deepseek/deepseek-r1-0528-qwen3-8b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "DeepSeek R1 0528 distilled into Qwen3 8B for mathematics, coding, and reasoning", "family": "deepseek-thinking", "license": "MIT", "limit": { "context": 131072, "output": 32000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete distilled checkpoint, not the R1 teacher.", "sources": [ "https://huggingface.co/api/models/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B" ], "total_billion": 8.19073536 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek R1 0528 Qwen3 8B", "open_weights": true, "release_date": "2025-05-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B" } ] }, "deepseek/deepseek-r1-distill-qwen-32b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "R1 reasoning distilled into Qwen 2.5 32B for efficient open-weight step-by-step problem solving", "family": "deepseek-thinking", "limit": { "context": 131072, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete distilled checkpoint, not the R1 teacher.", "sources": [ "https://huggingface.co/api/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" ], "total_billion": 32.763876352 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek-R1-Distill-Qwen-32B", "open_weights": true, "release_date": "2025-01-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" } ] }, "deepseek/deepseek-reasoner": { "benchmarks": [ { "date": "2025-10-03", "metric": "percent correct", "name": "Aider Polyglot", "score": 74.2, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek reasoning model for multi-step analysis, math, coding, and tools", "family": "deepseek-thinking", "knowledge": "2025-09", "last_updated": "2026-02-28", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "notes": "Historical scope: the archived official February 2026 pricing table explicitly maps deepseek-reasoner to DeepSeek-V3.2 (Thinking Mode), matching this local December 2025 release and linked checkpoint. Counts are Artificial Analysis's V3.2 architecture specification; not a claim about the current moving API alias or attribution of older benchmark runs.", "sources": [ "https://web.archive.org/web/20260204135805id_/https://api-docs.deepseek.com/quick_start/pricing/", "https://api-docs.deepseek.com/news/news251201", "https://artificialanalysis.ai/models/deepseek-v3-2" ], "total_billion": 685 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek Reasoner", "open_weights": true, "release_date": "2025-12-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2" } ] }, "deepseek/deepseek-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open DeepSeek MoE chat model for coding, math, and general reasoning", "family": "deepseek", "license": "DeepSeek Model License", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V3" ], "total_billion": 671 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek-V3", "open_weights": true, "release_date": "2024-12-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3" } ] }, "deepseek/deepseek-v3-0324": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "March 2025 checkpoint of DeepSeek-V3 with improved reasoning and coding", "family": "deepseek", "limit": { "context": 163840, "output": 163840 }, "metadata": { "modelatlas": { "parameters": { "notes": "OpenRouter explicitly maps its version-specific deepseek/deepseek-chat-v3-0324 entry to this checkpoint and reports 685B parameters. Component inclusion is unspecified; no active count inferred.", "sources": [ "https://openrouter.ai/api/v1/models", "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324" ], "total_billion": 685 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V3 0324", "open_weights": true, "release_date": "2025-03-24", "schema": 2, "weights": [ { "format": "safetensors", "label": "Model weights", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324" } ] }, "deepseek/deepseek-v3.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Hybrid-reasoning DeepSeek model with thinking and non-thinking modes", "family": "deepseek", "license": "MIT License", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V3.1" ], "total_billion": 671 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek-V3.1", "open_weights": true, "release_date": "2025-08-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1" } ] }, "deepseek/deepseek-v3.2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Hybrid-reasoning DeepSeek model with thinking and non-thinking modes, sparse attention, and tool-use", "family": "deepseek", "knowledge": "2024-07", "license": "MIT License", "limit": { "context": 128000, "output": 64000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 37, "notes": "Artificial Analysis explicitly reports these architecture counts for DeepSeek V3.2. Not inferred from FP8 tensors or inherited from an earlier DeepSeek version.", "sources": [ "https://artificialanalysis.ai/models/deepseek-v3-2", "https://huggingface.co/deepseek-ai/DeepSeek-V3.2" ], "total_billion": 685 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V3.2", "open_weights": true, "release_date": "2025-12-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2" } ] }, "deepseek/deepseek-v4-flash": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 79, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" }, { "metric": "EM", "name": "MMLU-Pro", "score": 86.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "SimpleQA-Verified", "score": 34.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Chinese SimpleQA", "score": 78.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "GPQA Diamond", "score": 88.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Humanity's Last Exam", "score": 34.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort; without tools" }, { "metric": "pass@1", "name": "LiveCodeBench", "score": 91.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "rating", "name": "Codeforces", "score": 3052, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "dataset": "February 2026", "metric": "pass@1", "name": "HMMT", "score": 94.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "IMOAnswerBench", "score": 88.4, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "MathArena Apex", "score": 33, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "MathArena Apex Shortlist", "score": 85.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "dataset": "1M context", "metric": "MMR", "name": "MRCR", "score": 78.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "dataset": "1M context", "metric": "accuracy", "name": "CorpusQA", "score": 60.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "accuracy", "name": "Terminal-Bench", "score": 56.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort", "version": "2.0" }, { "metric": "resolved", "name": "SWE-Bench Pro", "score": 52.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "resolved", "name": "SWE-Bench Multilingual", "score": 73.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "BrowseComp", "score": 73.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Humanity's Last Exam", "score": 45.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort; with tools" }, { "dataset": "public", "metric": "pass@1", "name": "MCP Atlas", "score": 69, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1395, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Toolathlon", "score": 47.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "variant": "preview checkpoint; max effort" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work", "family": "deepseek-flash", "knowledge": "2025-05", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 13, "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" ], "total_billion": 284 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Flash", "open_weights": true, "release_date": "2026-04-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" } ] }, "deepseek/deepseek-v4-flash-0423": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Initial DeepSeek V4 Flash snapshot for economical reasoning, coding, and million-token agent workloads", "family": "deepseek-flash", "knowledge": "2025-05", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 13, "notes": "Initial April 2026 checkpoint linked by this version-specific local entry.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" ], "total_billion": 284 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Flash 0423", "open_weights": true, "release_date": "2026-04-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" } ] }, "deepseek/deepseek-v4-flash-0731": { "benchmarks": [ { "metric": "pass@1", "name": "Terminal-Bench", "score": 82.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "variant": "max", "version": "2.1" }, { "date": "2026-07-31", "harness": "DeepSeek Harness minimal mode", "metric": "resolve rate", "name": "NL2Repo", "score": 54.2, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "date": "2026-07-31", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "CyberGym", "score": 76.7, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "date": "2026-07-31", "harness": "DeepSeek Harness minimal mode", "metric": "resolve rate", "name": "DeepSWE", "score": 54.4, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "date": "2026-07-31", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "Toolathlon-Verified", "score": 70.3, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "date": "2026-07-31", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "Agents' Last Exam", "score": 25.2, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "dataset": "public", "date": "2026-07-31", "metric": "success rate", "name": "AutomationBench", "score": 25.1, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "dataset": "internal", "date": "2026-07-31", "metric": "score", "name": "DSBench-FullStack", "score": 68.7, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "dataset": "internal", "date": "2026-07-31", "metric": "score", "name": "DSBench-Hard", "score": 59.6, "source": "https://api-docs.deepseek.com/updates/", "variant": "max effort" }, { "dataset": "113 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "pass@1", "name": "DeepSWE", "score": "53% ±4%", "source": "https://deepswe.datacurve.ai/", "version": "1.1" }, { "cost": 0.766472, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 18.687, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "average@3; per-run pass@1", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 4.29, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single run; pass@1; retry provider/infra errors", "version": "0.1.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding", "family": "deepseek-flash", "knowledge": "2025-05", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 13, "notes": "Explicit total and active counts in OpenRouter's deepseek/deepseek-v4-flash-0731 description, linked there to this exact July checkpoint.", "sources": [ "https://openrouter.ai/api/v1/models", "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731" ], "total_billion": 284 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Flash 0731", "open_weights": true, "release_date": "2026-07-31", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731" } ] }, "deepseek/deepseek-v4-flash-vision-exp": { "benchmarks": [ { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "Terminal-Bench", "score": 83.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort", "version": "2.1" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "NL2Repo", "score": 57.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "CyberGym", "score": 75.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "DeepSWE", "score": 59.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "Toolathlon-Verified", "score": 75.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "dataset": "internal", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "DSBench-Hard", "score": 63.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "dataset": "public", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "AutomationBench", "score": 25.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "variant": "max effort" }, { "metric": "pass@1", "name": "ApexBench", "score": 36.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, { "metric": "score", "name": "Agents' Last Exam", "score": 27.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, { "metric": "score", "name": "Chartography", "score": 64.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, { "metric": "pass@5", "name": "ZeroBench", "score": 35, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, { "dataset": "Chartography full task set", "effort": "Max reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 11.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "dataset": "GDPval (220 tasks)", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1554, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Experimental multimodal DeepSeek V4 Flash model for image understanding, coding, and agentic work", "family": "deepseek-flash", "last_updated": "2026-09-01", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Vision encoder", "notes": "411,842,560 BF16 weight/bias values across all 259 vision.* tensors, independently verified from the shard header and checked against the full index.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/model-00001-of-00048.safetensors", "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/model.safetensors.index.json" ], "total_billion": 0.41184256 }, { "name": "Vision-language aligner", "notes": "54,534,144 BF16 weight/bias values across all four aligner.* tensors, independently verified from the shard header and checked against the full index.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/model-00001-of-00048.safetensors", "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp/blob/6821d6ad3681a4b137b066b76094fa82ebd0a380/model.safetensors.index.json" ], "total_billion": 0.054534144 } ], "notes": "Only the unquantized vision encoder and aligner are counted. Quantized language tensors and DSpark weights are not used to infer a full multimodal total." } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "DeepSeek V4 Flash Vision Exp", "open_weights": true, "release_date": "2026-08-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" } ] }, "deepseek/deepseek-v4-pro": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 80.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 50.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 67.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 18, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 64.7, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high", "version": "2.1" }, { "metric": "EM", "name": "MMLU-Pro", "score": 87.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "SimpleQA-Verified", "score": 57.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Chinese SimpleQA", "score": 84.4, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "GPQA Diamond", "score": 90.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Humanity's Last Exam", "score": 37.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort; without tools" }, { "metric": "pass@1", "name": "LiveCodeBench", "score": 93.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "rating", "name": "Codeforces", "score": 3206, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "dataset": "February 2026", "metric": "pass@1", "name": "HMMT", "score": 95.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "IMOAnswerBench", "score": 89.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "MathArena Apex", "score": 38.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "MathArena Apex Shortlist", "score": 90.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "dataset": "1M context", "metric": "MMR", "name": "MRCR", "score": 83.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "dataset": "1M context", "metric": "accuracy", "name": "CorpusQA", "score": 62, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "accuracy", "name": "Terminal-Bench", "score": 67.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort", "version": "2.0" }, { "metric": "resolved", "name": "SWE-Bench Pro", "score": 55.4, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "resolved", "name": "SWE-Bench Multilingual", "score": 76.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "BrowseComp", "score": 83.4, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Humanity's Last Exam", "score": 48.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort; with tools" }, { "dataset": "public", "metric": "pass@1", "name": "MCP Atlas", "score": 73.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1554, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "metric": "pass@1", "name": "Toolathlon", "score": 51.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "variant": "preview checkpoint; max effort" }, { "dataset": "220 tasks across 44 occupations and 9 major industries", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1166, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 62.83185840707964, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 3.452522, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 11.111, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open MoE flagship with million-token context for coding and long agent runs", "family": "deepseek-thinking", "knowledge": "2025-05", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 49, "notes": "Official total reported as 1.6T, converted to B.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" ], "total_billion": 1600 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Pro", "open_weights": true, "release_date": "2026-04-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ] }, "deepseek/deepseek-v4-pro-0423": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek V4 Pro initial snapshot with million-token context and support for thinking and non-thinking modes", "family": "deepseek-thinking", "knowledge": "2025-05", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 49, "notes": "Initial April 2026 checkpoint; official total reported as 1.6T, converted to B.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" ], "total_billion": 1600 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Pro 0423", "open_weights": true, "release_date": "2026-04-23", "schema": 2 }, "deepseek/deepseek-v4-pro-0813": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "without tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 60, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "with tools" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "Terminal-Bench", "score": 87.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "max effort", "version": "2.1" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "NL2Repo", "score": 61.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "max effort" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "CyberGym", "score": 83.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "max effort" }, { "effort": "Max", "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "DeepSWE", "score": 62.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813", "variant": "temperature=1.0; top_p=0.95" }, { "metric": "score", "name": "Toolathlon-Verified", "score": 74.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "metric": "score", "name": "Agents' Last Exam", "score": 25.7, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "dataset": "public", "metric": "score", "name": "AutomationBench", "score": 31.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "dataset": "internal", "metric": "score", "name": "DSBench-FullStack", "score": 71.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "dataset": "internal", "metric": "score", "name": "DSBench-Hard", "score": 67.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" }, { "cost": 3.311817, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 14.141, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1; three runs; ±1 standard error", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 4.29, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1 single run; retry infrastructure/provider errors only", "version": "0.1.0" }, { "cost": 1.7691338076407441, "cost_source": "https://artificialanalysis.ai/models/deepseek-v4-pro", "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 23.0632981493003, "source": "https://artificialanalysis.ai/models/releases/deepseek-v4-pro", "version": "5.0.1" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1066, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "non-reasoning", "version": "2.1" }, { "cost": 2.499691, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 52.974, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 0.9769879356486032, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 12.5675939779921, "source": "https://artificialanalysis.ai/models/deepseek-v4-pro-non-reasoning", "variant": "Non-reasoning", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek V4 Pro snapshot with million-token context and support for thinking and non-thinking modes", "family": "deepseek-thinking", "last_updated": "2026-08-22", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 49, "notes": "Artificial Analysis explicitly identifies this page as DeepSeek V4 Pro 0813 (Max), released August 2026, and reports 1.6T total and 49B active. These are its published target-model architecture counts, not an inferred sum including the separately described DSpark decoding module.", "sources": [ "https://artificialanalysis.ai/models/deepseek-v4-pro", "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" ], "total_billion": 1600 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "DeepSeek V4 Pro 0813", "open_weights": true, "release_date": "2026-08-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813" } ] }, "deepseek/deepseek-v4.1-flash": { "benchmarks": [ { "metric": "pass@1", "name": "GPQA Diamond", "score": 90.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "MathArena Apex", "score": 65.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "dataset": "full set", "metric": "pass@1", "name": "Humanity's Last Exam", "score": 36.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "dataset": "text-only subset", "metric": "pass@1", "name": "Humanity's Last Exam", "score": 39.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "rating", "name": "Codeforces", "score": 3471, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "harness": "DeepSeek Harness minimal mode", "metric": "pass@1", "name": "Terminal-Bench", "score": 90.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "DeepSeek Harness minimal mode", "metric": "pass@1", "name": "Terminal-Bench", "score": 30.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "3.0" }, { "harness": "DeepSeek Harness minimal mode", "metric": "pass@1", "name": "Terminal-Bench", "score": 31.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "4.0.0" }, { "harness": "mini-swe-agent", "metric": "resolved", "name": "DeepSWE", "score": 74.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "DeepSeek Harness minimal mode", "metric": "almost@1", "name": "ProgramBench", "score": 20.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "harness": "DeepSeek Harness minimal mode", "metric": "score", "name": "NL2Repo", "score": 64.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "CyberGym", "score": 88.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "ExploitGym", "score": 15.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "AutomationBench", "score": 54.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "Agents' Last Exam", "score": 31.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SEC-Bench Pro", "score": 62.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100" }, { "metric": "pass@1", "name": "Humanity's Last Exam", "score": 63.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100; with tools" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Chartography", "score": 78.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100; with tools" }, { "harness": "Claude Code", "metric": "pass@1", "name": "BabyVision", "score": 89.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100; with tools" }, { "dataset": "main", "harness": "Claude Code", "metric": "pass@5", "name": "ZeroBench", "score": 49.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100; with tools" }, { "harness": "Claude Code", "metric": "resolved", "name": "DeepSWE", "score": 69.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 88.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "Codex", "metric": "resolved", "name": "DeepSWE", "score": 65.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "Codex", "metric": "pass@1", "name": "Terminal-Bench", "score": 84.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "OpenCode", "metric": "resolved", "name": "DeepSWE", "score": 65.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "OpenCode", "metric": "pass@1", "name": "Terminal-Bench", "score": 85.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "Pi", "metric": "resolved", "name": "DeepSWE", "score": 66.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "Pi", "metric": "pass@1", "name": "Terminal-Bench", "score": 86.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 90.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "DeepSeek Harness minimal mode", "metric": "resolved", "name": "DeepSWE", "score": 72.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "DeepSeek Harness standard mode", "metric": "resolved", "name": "DeepSWE", "score": 70.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "DeepSeek Harness standard mode", "metric": "pass@1", "name": "Terminal-Bench", "score": 85.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "harness": "DeepSeek Harness PTC mode", "metric": "resolved", "name": "DeepSWE", "score": 67.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "1.1" }, { "harness": "DeepSeek Harness PTC mode", "metric": "pass@1", "name": "Terminal-Bench", "score": 85.8, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash", "variant": "reasoning_effort=100", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Max", "harness": "Stirrup shell and web agent loop", "metric": "Elo rating", "name": "GDPval-AA", "score": 1600, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Artificial Analysis leaderboard anchor", "version": "2.1" }, { "dataset": "100 tasks; 10 runs per task", "effort": "Max reasoning", "harness": "Inspect AI; Gemini 3.5 Flash judge", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 11.6, "source": "https://surgehq.ai/benchmarks/chartography" }, { "cost": 0.499473, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 19.697, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "DeepSeek V4.1 Flash model for reasoning and agentic coding", "family": "deepseek-flash", "knowledge": "2025-05", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 16, "notes": "Decode-stage active count only; the official card separately reports 8B per token during prefill. Complete total omitted: 552B is explicitly backbone-only and the card additionally describes Engram memory and visual modules.", "sources": [ "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" ] } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "DeepSeek V4.1 Flash", "open_weights": true, "release_date": "2026-09-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash" } ] }, "deepseek/deepseek-v4.1-flash-1017": { "benchmarks": [ { "dataset": "full set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42.6, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "variant": "without tools", "version": "unspecified" }, { "dataset": "text-only subset", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "variant": "without tools", "version": "unspecified" }, { "harness": "Claude Code", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 42.3, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "version": "4.0.0" }, { "dataset": "DeepSWE v1.1 (113 tasks)", "harness": "Mini-SWE-V2", "metric": "pass@1", "name": "DeepSWE", "score": 77.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "version": "1.1" }, { "name": "ProgramBench", "score": 29.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, { "name": "NL2Repo", "score": 68.2, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, { "metric": "task_completed_correctly", "name": "AutomationBench", "score": 60.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "version": "1.0.6" }, { "name": "Agents' Last Exam", "score": 35.4, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, { "name": "Apex Bench", "score": 72.1, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "version": "1.1" }, { "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 83.0, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "variant": "with tools", "version": "unspecified" }, { "name": "BabyVision", "score": 92.5, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017", "variant": "with tools" }, { "dataset": "main", "metric": "pass@5", "name": "ZeroBench", "score": 51.0, "source": "https://api-docs.deepseek.com/updates/", "variant": "with tools" }, { "dataset": "harborframework/terminal-bench-science; terminalbenchscience_0_1 (70 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 12.9, "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/discussions/40", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "220 GDPval tasks", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1746, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 0.65747, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 61.257, "source": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash-1017", "version": "2.2" }, { "cost": 0.4254202455035728, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 18.5131594039446, "source": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017-non-reasoning", "variant": "Non-reasoning", "version": "5.0.1" }, { "cost": 0.45619449806633267, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 28.6437391540402, "source": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017-high", "version": "5.0.1" }, { "cost": 0.60210514783598, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 32.5896393532027, "source": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017", "version": "5.0.1" }, { "cost": 0.539359, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 31.818, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.3, "source": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 25.3, "source": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "October 17 update of DeepSeek V4.1 Flash with improved reasoning, coding, agentic tasks, and visual understanding", "family": "deepseek-flash", "knowledge": "2025-05", "license": "MIT", "limit": { "context": 1000000, "output": 384000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 16, "notes": "Artificial Analysis reports 16B active for this exact snapshot, without a separate prefill count. Complete total omitted: the family's 552B figure is backbone-only, not an inclusive model total.", "sources": [ "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017", "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017" ] } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "DeepSeek V4.1 Flash 1017", "open_weights": true, "release_date": "2026-10-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017" } ] }, "elevenlabs/eleven_flash_v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Ultra-low-latency English text-to-speech model for real-time use", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven Flash v2", "open_weights": false, "release_date": "2024-12-19", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_flash_v2_5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Ultra-low-latency multilingual text-to-speech model for real-time use", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven Flash v2.5", "open_weights": false, "release_date": "2024-12-19", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_multilingual_v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lifelike multilingual text-to-speech model for long-form narration in 29 languages", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven Multilingual v2", "open_weights": false, "release_date": "2023-08-22", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_turbo_v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency English text-to-speech model, superseded by Flash v2", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven Turbo v2", "open_weights": false, "release_date": "2023-11-09", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_turbo_v2_5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency multilingual text-to-speech model, superseded by Flash v2.5", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven Turbo v2.5", "open_weights": false, "release_date": "2024-07-19", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive text-to-speech model with inline audio tags across 70+ languages", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven v3", "open_weights": false, "release_date": "2025-08-20", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_v3_conversational": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Real-time variant of Eleven v3 tuned for conversational agents", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven v3 Conversational", "open_weights": false, "release_date": "2026-02-09", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_v4": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive multilingual text-to-speech model with audio tags and voice cloning across 90+ languages", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven v4", "open_weights": false, "release_date": "2026-09-28", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/eleven_v4_turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency Eleven v4 text-to-speech model for real-time agents", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/wav", "audio/opus", "audio/basic", "audio/alaw" ] }, "name": "Eleven v4 Turbo", "open_weights": false, "release_date": "2026-09-28", "schema": 2, "type": "speech", "v2_only": true }, "elevenlabs/scribe_v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with word timestamps, speaker diarization, and audio-event tagging in 90+ languages", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/aac", "audio/aiff", "audio/flac", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/opus", "audio/wav", "audio/webm", "video/mp4", "video/mpeg", "video/quicktime", "video/webm", "video/x-msvideo", "video/x-matroska", "video/x-ms-wmv", "video/x-flv", "video/3gpp" ], "output": [ "text" ] }, "name": "Scribe v2", "open_weights": false, "release_date": "2026-01-12", "schema": 2, "type": "transcription", "v2_only": true }, "elevenlabs/scribe_v2_medical": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Scribe v2 speech-to-text model specialized for medical and clinical audio", "family": "elevenlabs", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/aac", "audio/aiff", "audio/flac", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/opus", "audio/wav", "audio/webm", "video/mp4", "video/mpeg", "video/quicktime", "video/webm", "video/x-msvideo", "video/x-matroska", "video/x-ms-wmv", "video/x-flv", "video/3gpp" ], "output": [ "text" ] }, "name": "Scribe v2 Medical", "open_weights": false, "release_date": "2026-09-11", "schema": 2, "type": "transcription", "v2_only": true }, "essentialai/rnj-1-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open 8B Essential AI model for code, STEM reasoning, and agentic tool use", "family": "rnj", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Uses the architecture table's 8.3B total, rather than the introductory 8B size designation.", "sources": [ "https://huggingface.co/EssentialAI/rnj-1-instruct" ], "total_billion": 8.3 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Rnj-1 Instruct", "open_weights": true, "release_date": "2025-12-04", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/EssentialAI/rnj-1-instruct" } ] }, "fireworks/ember-1": { "benchmarks": [ { "dataset": "113 tasks", "metric": "score", "name": "DeepSWE", "score": 75.2, "source": "https://fireworks.ai/blog/ember-1", "version": "1.1" }, { "dataset": "113 tasks", "metric": "score", "name": "DeepSWE", "score": 67.0, "source": "https://fireworks.ai/specialized-intelligence-index/benchmarks/code-generation/deepswe-v1-1", "variant": "avg@3 (three-run mean)", "version": "1.1" }, { "cost": 19.484094, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 20.707, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (three full runs; each run pass@1)", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": "5.71 ± 2.79", "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single-run pass@1", "version": "0.1.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Specialized model from Fireworks built on Kimi K3, producing shorter reasoning traces with approximately 40% fewer tokens while maintaining comparable quality", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/gif", "image/bmp", "image/tiff", "image/x-portable-pixmap" ], "output": [ "text" ] }, "name": "Ember-1", "open_weights": false, "release_date": "2026-09-22", "schema": 2 }, "fish-audio/drama-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-speech model directed with plain-language performance notes", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/wav", "audio/pcm", "audio/opus" ] }, "name": "Drama 3", "open_weights": false, "release_date": "2026-10-13", "schema": 2, "type": "speech", "v2_only": true }, "fish-audio/s1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text-to-speech model with parenthetical emotion controls", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/wav", "audio/pcm", "audio/opus" ] }, "name": "S1", "open_weights": false, "release_date": "2025-06", "schema": 2, "type": "speech", "v2_only": true }, "fish-audio/s2-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual text-to-speech model with bracket emotion cues and multi-speaker dialogue", "license": "Fish Audio Research License", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete dual-AR checkpoint, rather than inferring a sum from rounded component sizes.", "sources": [ "https://huggingface.co/api/models/fishaudio/s2-pro" ], "total_billion": 4.561852416 } } }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/wav", "audio/pcm", "audio/opus" ] }, "name": "S2 Pro", "open_weights": true, "release_date": "2026-03", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/fishaudio/s2-pro" } ] }, "fish-audio/s2.1-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Production text-to-speech model improving on S2 Pro in quality, latency, and throughput", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/wav", "audio/pcm", "audio/opus" ] }, "name": "S2.1 Pro", "open_weights": false, "release_date": "2026-06-23", "schema": 2, "type": "speech", "v2_only": true }, "fish-audio/transcribe-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model for short recordings with language detection and optional timestamps", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/aac", "audio/mp4", "audio/flac", "audio/ogg" ], "output": [ "text" ] }, "name": "Transcribe 1", "open_weights": false, "release_date": "2025-09", "schema": 2, "type": "transcription", "v2_only": true }, "fish-audio/transcribe-1-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with speaker markers and emotion and vocal-event cues", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/aac", "audio/mp4", "audio/flac", "audio/ogg", "audio/webm", "video/x-matroska", "video/quicktime" ], "output": [ "text" ] }, "name": "Transcribe 1 Pro", "open_weights": false, "release_date": "2026-09-18", "schema": 2, "type": "transcription", "v2_only": true }, "google/chirp-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual speech-to-text model with diarization and automatic language detection", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Chirp 3", "open_weights": false, "release_date": "2025-08-29", "schema": 2, "type": "transcription", "v2_only": true }, "google/deep-research-max-preview-04-2026": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Maximum-comprehensiveness agentic researcher for multi-step investigation, synthesis, and cited reports", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text", "image" ] }, "name": "Deep Research Max Preview", "open_weights": false, "release_date": "2026-04-21", "schema": 2 }, "google/deep-research-preview-04-2026": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Agentic model for autonomous multi-step research, synthesis, and cited reports", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text", "image" ] }, "name": "Gemini Deep Research Preview", "open_weights": false, "release_date": "2026-04-21", "schema": 2 }, "google/diffusiongemma-26b-a4b-it": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 11.0, "source": "https://ai.google.dev/gemma/docs/diffusiongemma/model_card", "variant": "no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 11.9, "source": "https://ai.google.dev/gemma/docs/diffusiongemma/model_card", "variant": "with search" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Gemma 4 based discrete text diffusion model for fast chat, reasoning, and tool use", "family": "gemma", "knowledge": "2025-01", "last_updated": "2026-07-15", "license": "Apache 2.0", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.8, "notes": "Official per-token active count. Complete total omitted: the table separately lists the vision encoder, and the Hub's safetensors total disagrees with its dtype parameter count.", "sources": [ "https://huggingface.co/google/diffusiongemma-26B-A4B-it" ] } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "DiffusionGemma 26B A4B IT", "open_weights": true, "release_date": "2026-06-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/diffusiongemma-26B-A4B-it" } ] }, "google/embeddinggemma-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multimodal embedding model for text, code, images, video and audio", "family": "gemma", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text", "image", "audio", "video" ], "output": [ "embedding" ] }, "name": "EmbeddingGemma 2", "open_weights": true, "release_date": "2026-10-06", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/embeddinggemma-2" } ] }, "google/embeddinggemma-300m": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Compact multilingual open text embedding model with Matryoshka output dimensions", "family": "gemma", "license": "Gemma Terms of Use", "limit": { "context": 2048, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "EmbeddingGemma 300M", "open_weights": true, "release_date": "2025-09-04", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/embeddinggemma-300m" } ] }, "google/gemini-2.0-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Earlier Gemini Flash workhorse for responsive multimodal apps and tool use", "family": "gemini-flash", "knowledge": "2024-06", "limit": { "context": 1048576, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.0 Flash", "open_weights": false, "release_date": "2024-12-11", "schema": 2 }, "google/gemini-2.0-flash-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Low-latency Gemini model for high-volume multimodal and agent workloads", "family": "gemini-flash-lite", "knowledge": "2024-06", "limit": { "context": 1048576, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.0 Flash-Lite", "open_weights": false, "release_date": "2024-12-11", "schema": 2 }, "google/gemini-2.5-computer-use-preview-10-2025": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Specialized Gemini 2.5 model for browser-control agents that automate UI tasks", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 128000, "output": 64000 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemini 2.5 Computer Use Preview", "open_weights": false, "release_date": "2025-10-07", "schema": 2 }, "google/gemini-2.5-flash": { "benchmarks": [ { "date": "2025-05-25", "metric": "percent correct", "name": "Aider Polyglot", "score": 55.1, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-06-02", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 22.2, "source": "https://openrouter.ai/google/gemini-2.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "percent correct", "name": "SciCode", "score": 39.4, "source": "https://openrouter.ai/google/gemini-2.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 13.6, "source": "https://openrouter.ai/google/gemini-2.5-flash/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Gemini workhorse for multimodal apps where latency and price matter", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Flash", "open_weights": false, "release_date": "2025-06-17", "schema": 2 }, "google/gemini-2.5-flash-image": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Nano Banana image model for fast generation, edits, and character-consistent assets", "family": "gemini-flash", "knowledge": "2024-06", "limit": { "context": 32768, "output": 32768 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana", "open_weights": false, "release_date": "2025-08-26", "schema": 2, "type": "image" }, "google/gemini-2.5-flash-lite": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 9.5, "source": "https://openrouter.ai/google/gemini-2.5-flash-lite/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 19.3, "source": "https://openrouter.ai/google/gemini-2.5-flash-lite/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 4.5, "source": "https://openrouter.ai/google/gemini-2.5-flash-lite/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lean Gemini 2.5 lane for cheap multimodal traffic and quick agents", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Flash-Lite", "open_weights": false, "release_date": "2025-06-17", "schema": 2 }, "google/gemini-2.5-flash-lite-preview-06-17": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "First preview of Gemini 2.5 Flash-Lite, the low-cost, low-latency 2.5 model", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Flash-Lite Preview 06-17", "open_weights": false, "release_date": "2025-06-17", "schema": 2 }, "google/gemini-2.5-flash-lite-preview-09-2025": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "September 2025 preview of Gemini 2.5 Flash-Lite with better instruction following and multimodal quality", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Flash-Lite Preview 09-2025", "open_weights": false, "release_date": "2025-09-25", "schema": 2 }, "google/gemini-2.5-flash-native-audio-preview-12-2025": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Gemini 2.5 Flash Live API model with native audio for real-time conversational agents", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video", "audio/pcm" ], "output": [ "text", "audio/pcm" ] }, "name": "Gemini 2.5 Flash Native Audio Preview 12-2025", "open_weights": false, "release_date": "2025-12-12", "schema": 2, "type": "realtime", "v2_only": true }, "google/gemini-2.5-flash-preview-05-20": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 10.96, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Preview snapshot of Gemini 2.5 Flash with controllable thinking budgets", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Flash Preview 05-20", "open_weights": false, "release_date": "2025-05-20", "schema": 2 }, "google/gemini-2.5-flash-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "gemini-flash", "knowledge": "2025-01", "last_updated": "2025-12-10", "limit": { "context": 32768, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "audio/l16" ] }, "name": "Gemini 2.5 Flash TTS", "open_weights": false, "release_date": "2025-09-30", "schema": 2, "type": "speech" }, "google/gemini-2.5-pro": { "benchmarks": [ { "date": "2025-06-06", "metric": "percent correct", "name": "Aider Polyglot", "score": 83.1, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-06-02", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 32, "source": "https://openrouter.ai/google/gemini-2.5-pro/benchmarks" }, { "date": "2026-06-02", "metric": "percent correct", "name": "SciCode", "score": 42.8, "source": "https://openrouter.ai/google/gemini-2.5-pro/benchmarks" }, { "date": "2026-06-02", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 26.5, "source": "https://openrouter.ai/google/gemini-2.5-pro/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Google's proven reasoning model for coding, math, and multimodal analysis", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 2.5 Pro", "open_weights": false, "release_date": "2025-06-17", "schema": 2 }, "google/gemini-2.5-pro-preview": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 21.64, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Advanced Gemini model for complex reasoning, coding, and multimodal analysis", "family": "gemini", "knowledge": "2025-01-31", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "application/pdf", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "text", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm" ], "output": [ "text" ] }, "name": "Gemini 2.5 Pro Preview 06-05", "open_weights": false, "release_date": "2025-06-05", "schema": 2 }, "google/gemini-2.5-pro-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "gemini-pro", "knowledge": "2025-01", "last_updated": "2025-12-10", "limit": { "context": 32768, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "audio/l16" ] }, "name": "Gemini 2.5 Pro TTS", "open_weights": false, "release_date": "2025-09-30", "schema": 2, "type": "speech" }, "google/gemini-3-flash-preview": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 34.63, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 8.2, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 10, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 30.3, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3 Flash Preview", "open_weights": false, "release_date": "2025-12-17", "schema": 2 }, "google/gemini-3-pro-image": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Nano Banana Pro for higher-fidelity image generation and design-heavy edits", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 65536, "output": 32768 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana Pro", "open_weights": false, "release_date": "2026-05-28", "schema": 2, "type": "image" }, "google/gemini-3-pro-image-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Nano Banana Pro for higher-fidelity image generation and design-heavy edits", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 65536, "output": 32768 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana Pro Preview", "open_weights": false, "release_date": "2025-11-20", "schema": 2, "type": "image" }, "google/gemini-3-pro-preview": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 43.3, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 37.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Preview Gemini flagship for complex reasoning, coding, and rich multimodal prompts", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3 Pro Preview", "open_weights": false, "release_date": "2025-11-18", "schema": 2 }, "google/gemini-3.1-flash-image": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "application/pdf" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana 2", "open_weights": false, "release_date": "2026-05-28", "schema": 2, "type": "image" }, "google/gemini-3.1-flash-image-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 65536, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "application/pdf" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana 2 Preview", "open_weights": false, "release_date": "2026-02-26", "schema": 2, "type": "image" }, "google/gemini-3.1-flash-lite": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Low-latency Gemini model for high-volume multimodal and agent workloads", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.1 Flash Lite", "open_weights": false, "release_date": "2026-05-07", "schema": 2 }, "google/gemini-3.1-flash-lite-image": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fastest, most cost-efficient Gemini image model for high-volume 1K generation and editing", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 65536, "output": 4096 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana 2 Lite", "open_weights": false, "release_date": "2026-06-30", "schema": 2, "type": "image" }, "google/gemini-3.1-flash-lite-preview": { "benchmarks": [ { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 6, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-5-flash-minimal-vs-gemini-3-1-flash-lite-preview", "version": "5.0.1" }, { "cost": 0.19445134743611814, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 6.15684082814488, "source": "https://artificialanalysis.ai/models/gemini-3-1-flash-lite-preview", "version": "5.0.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.64, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Low-latency Gemini model for high-volume multimodal and agent workloads", "family": "gemini-flash-lite", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.1 Flash Lite Preview", "open_weights": false, "release_date": "2026-03-03", "schema": 2 }, "google/gemini-3.1-flash-live-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "High-quality, low-latency Live API model for real-time dialogue and voice-first AI applications", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video", "audio/pcm" ], "output": [ "text", "audio/pcm" ] }, "name": "Gemini 3.1 Flash Live Preview", "open_weights": false, "release_date": "2026-03-26", "schema": 2, "type": "realtime" }, "google/gemini-3.1-flash-tts-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Low-latency speech generation with steerable prompts and expressive audio tags", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 8192, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "audio/l16" ] }, "name": "Gemini 3.1 Flash TTS Preview", "open_weights": false, "release_date": "2026-04-15", "schema": 2, "type": "speech" }, "google/gemini-3.1-pro-preview": { "benchmarks": [ { "date": "2026-05-28", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 54.2, "source": "https://www.anthropic.com/news/claude-opus-4-8" }, { "date": "2026-05-28", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 70.3, "source": "https://www.anthropic.com/news/claude-opus-4-8", "version": "2.1" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 46.1, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 13.5, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Gemini CLI", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 33.81, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 29.84, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Gemini CLI", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 43, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "harness": "Gemini CLI", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 45.6, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "dataset": "hard-aa", "harness": "Gemini CLI", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 15.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high" }, { "harness": "Gemini CLI", "metric": "pass@1", "name": "Terminal-Bench", "score": 68.3, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "high", "version": "2.1" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GPQA Diamond", "score": 94.3, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "full set, text + MM", "date": "2026-05-19", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.4, "source": "https://deepmind.google/models/gemini/flash/" }, { "date": "2026-05-19", "metric": "accuracy", "name": "ARC-AGI-2", "score": 77.1, "source": "https://deepmind.google/models/gemini/flash/" }, { "date": "2026-05-19", "metric": "accuracy", "name": "MMMU Pro", "score": 80.5, "source": "https://deepmind.google/models/gemini/flash/", "variant": "no tools" }, { "date": "2026-05-19", "metric": "success rate", "name": "MCP Atlas", "score": 78.2, "source": "https://deepmind.google/models/gemini/flash/" }, { "date": "2026-05-19", "metric": "success rate", "name": "OSWorld-Verified", "score": 76.2, "source": "https://deepmind.google/models/gemini/flash/" }, { "date": "2026-05-19", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 83.3, "source": "https://deepmind.google/models/gemini/flash/", "variant": "no tools" }, { "date": "2026-05-19", "metric": "Elo", "name": "GDPval-AA", "score": 1314, "source": "https://deepmind.google/models/gemini/flash/" }, { "cost": 2.1434045568888886, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 11.72566371681416, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 4.111957, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 2.525, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "high", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.44, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning-first Gemini preview for agentic coding and complex problem solving", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.1 Pro Preview", "open_weights": false, "release_date": "2026-02-19", "schema": 2 }, "google/gemini-3.1-pro-preview-customtools": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Advanced Gemini model for complex reasoning, coding, and multimodal analysis", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.1 Pro Preview Custom Tools", "open_weights": false, "release_date": "2026-02-19", "schema": 2 }, "google/gemini-3.5-flash": { "benchmarks": [ { "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 76.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/", "version": "2.1" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 55.1, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/", "variant": "single attempt" }, { "metric": "success rate", "name": "MCP Atlas", "score": 83.6, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "metric": "success rate", "name": "Toolathlon", "score": 56.5, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "metric": "success rate", "name": "OSWorld-Verified", "score": 78.4, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "metric": "accuracy", "name": "MMMU Pro", "score": 83.6, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/", "variant": "no tools" }, { "metric": "accuracy", "name": "CharXiv Reasoning", "score": 84.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/", "variant": "no tools" }, { "dataset": "full set, text + MM", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 40.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "metric": "accuracy", "name": "ARC-AGI-2", "score": 72.1, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1656, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash/" }, { "cost": 3.4467114588691796, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 36.06194690265487, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 5.981757, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 6.061, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Gemini model balancing multimodal reasoning, tool use, and cost", "family": "gemini-flash", "knowledge": "2025-01", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.5 Flash", "open_weights": false, "release_date": "2026-05-19", "schema": 2 }, "google/gemini-3.5-flash-lite": { "benchmarks": [ { "date": "2026-07-21", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 54.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/" }, { "date": "2026-07-21", "harness": "Terminus 2", "metric": "accuracy", "name": "Terminal-Bench", "score": 54.0, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "version": "2.1" }, { "date": "2026-07-21", "metric": "average position score", "name": "MLE-Bench", "score": 39.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/" }, { "date": "2026-07-21", "metric": "Elo", "name": "GDPval-AA", "score": 1140, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "version": "v2" }, { "date": "2026-07-21", "metric": "success rate", "name": "OSWorld-Verified", "score": 74.0, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/" }, { "date": "2026-07-21", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 74.5, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "variant": "no tools" }, { "date": "2026-07-21", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 76.5, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "variant": "with tools" }, { "date": "2026-07-21", "metric": "accuracy", "name": "GDM-MRCR", "score": 72.2, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "variant": "128k average, 8-needle", "version": "v2" }, { "date": "2026-07-21", "metric": "accuracy", "name": "GDM-MRCR", "score": 21.3, "source": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "variant": "1M pointwise, 8-needle", "version": "v2" }, { "dataset": "Held-out private evaluation tasks (600+ tasks)", "effort": "Low", "harness": "API mode", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 0.0, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "Held-out private evaluation tasks (600+ tasks)", "effort": "Medium", "harness": "API mode", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 3.96, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "Held-out private evaluation tasks (600+ tasks)", "effort": "High", "harness": "API mode", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 3.35, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "SurgeAI Chartography (100 tasks; 20 trials per task)", "effort": "Minimal reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 10.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "no tools" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 12, "source": "https://artificialanalysis.ai/models/gemini-3-5-flash-lite", "version": "5.0.1" }, { "cost": 0.6929018557975878, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 11.6563462044212, "source": "https://artificialanalysis.ai/models/gemini-3-5-flash-lite", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Gemini model balancing multimodal reasoning, tool use, and cost", "family": "gemini-flash-lite", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.5 Flash Lite", "open_weights": false, "release_date": "2026-07-21", "schema": 2 }, "google/gemini-3.5-live-translate-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency audio-to-audio model for real-time speech translation across 70+ languages", "family": "gemini-pro", "knowledge": "2025-01", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "audio/pcm" ], "output": [ "audio/pcm", "text" ] }, "name": "Gemini 3.5 Live Translate Preview", "open_weights": false, "release_date": "2026-06-09", "schema": 2, "type": "realtime" }, "google/gemini-3.5-transcribe": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Speech-to-text model with language detection, speaker diarization, word timestamps, and custom vocabulary", "family": "gemini", "limit": { "context": 98304, "output": 32768 }, "modalities": { "input": [ "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm" ], "output": [ "text" ] }, "name": "Gemini 3.5 Transcribe", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "transcription", "v2_only": true }, "google/gemini-3.5-transcribe-live": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Speech transcription model for accurate audio-to-text and captioning workflows", "family": "gemini", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/pcm" ], "output": [ "text" ] }, "name": "Gemini 3.5 Transcribe Live", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "transcription" }, "google/gemini-3.6-flash": { "benchmarks": [ { "date": "2026-07-21", "harness": "Antigravity", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 58.7, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/" }, { "date": "2026-07-21", "metric": "resolve rate", "name": "DeepSWE", "score": 49.0, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "variant": "high reasoning", "version": "1.1" }, { "date": "2026-07-21", "harness": "Terminus 2", "metric": "accuracy", "name": "Terminal-Bench", "score": 78.0, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "version": "2.1" }, { "dataset": "Partial 30", "date": "2026-07-21", "metric": "average position score", "name": "MLE-Bench", "score": 63.9, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/" }, { "date": "2026-07-21", "metric": "Elo", "name": "GDPval-AA", "score": 1421, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "version": "v2" }, { "date": "2026-07-21", "metric": "success rate", "name": "OSWorld-Verified", "score": 83.0, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/" }, { "date": "2026-07-21", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 85.2, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "variant": "no tools" }, { "date": "2026-07-21", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 89.4, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "variant": "with tools" }, { "date": "2026-07-21", "metric": "accuracy", "name": "GDM-MRCR", "score": 91.8, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "variant": "128k average, 8-needle", "version": "v2" }, { "date": "2026-07-21", "metric": "accuracy", "name": "GDM-MRCR", "score": 54.0, "source": "https://deepmind.google/models/evals-methodology/gemini-3-6-flash/", "variant": "1M pointwise, 8-needle", "version": "v2" }, { "dataset": "Private set", "name": "AutomationBench", "score": 17.0, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/" }, { "dataset": "SurgeAI Chartography (100 tasks; 20 trials per task)", "effort": "Medium reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 34.0, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "no tools" }, { "dataset": "Artificial Analysis May 2025 text-only subset (2,158 questions)", "effort": "High reasoning", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 38.0, "source": "https://artificialanalysis.ai/articles/gemini-3-6-flash-3-5-flash-lite-halving-time", "variant": "no tools; pass@1" }, { "dataset": "Terminal-Bench Science v0.1.0 (70 tasks)", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 4.29, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1; single run", "version": "0.1.0" }, { "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 46.68141592920354, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Gemini model balancing multimodal reasoning, tool use, and cost", "family": "gemini-flash", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.6 Flash", "open_weights": false, "release_date": "2026-07-21", "schema": 2 }, "google/gemini-3.7-flash": { "benchmarks": [ { "metric": "score", "name": "FrontierCode", "score": 43.6, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "version": "1.1 Main" }, { "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.3, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "variant": "self-computed; LiteLLM 1.96; high thinking", "version": "1.1" }, { "cost": 2.0251351826880533, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.48672566371683, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "metric": "accuracy", "name": "Terminal-Bench", "score": 85.8, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "version": "2.1" }, { "cost": 8.705677, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 12.121, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1; 3 runs; ±1 standard error", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 7.14, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single-run pass@1; infrastructure/provider errors retried", "version": "0.1.0" }, { "dataset": "private set", "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 30.4, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "version": "1.0.6" }, { "dataset": "100-task full task set; 10 runs per task; Gemini 3.5 Flash judge", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 40.4, "source": "https://surgehq.ai/benchmarks/chartography" }, { "dataset": "2,158 text-only questions from the original HLE set", "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 48, "source": "https://artificialanalysis.ai/models/comparisons/claude-fable-5-1-high-vs-gemini-3-7-flash" }, { "metric": "accuracy", "name": "GDP.pdf", "score": 34.0, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/" }, { "metric": "accuracy", "name": "GDM-MRCR", "score": 97.0, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "variant": "128k average, 8-needle", "version": "v2" }, { "cost": 2.1762860351216813, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.2654867256637, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 1.8322645629424779, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.76106194690266, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "High-efficiency Gemini model for agentic workflows, coding, and multimodal reasoning", "family": "gemini-flash", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.7 Flash", "open_weights": false, "release_date": "2026-08-13", "schema": 2 }, "google/gemini-3.8-flash": { "benchmarks": [ { "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.7, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "high thinking", "version": "1.1" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1545, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "version": "2" }, { "metric": "pass@1", "name": "Finance Agent", "score": 61.4, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "version": "2" }, { "metric": "all-pass rate", "name": "Harvey Legal Agent Benchmark", "score": 10, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/" }, { "harness": "Terminus 2", "metric": "pass@1", "name": "Terminal-Bench", "score": 89.4, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "version": "2.1" }, { "metric": "pass@1", "name": "Terminal-Bench", "score": 19.1, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "version": "4.0.0" }, { "metric": "all-pass rate", "name": "GDP.PDF", "score": 35, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/" }, { "metric": "pass@1", "name": "CharXiv Reasoning", "score": 86.2, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "without tools" }, { "metric": "pass@1", "name": "LVBench", "score": 87.8, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "agentic" }, { "metric": "pass@1", "name": "LVBench", "score": 87.1, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "static; 1024 frames; without tools" }, { "dataset": "1811 verified and revised items", "metric": "accuracy", "name": "HLE-Verified", "score": 54.9, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/" }, { "dataset": "before 2026-08-08 patch", "harness": "Gemini CUA", "metric": "partial score", "name": "OSWorld", "score": 59, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "batched tool calls; best of 3 runs; 500 steps", "version": "2.0" }, { "dataset": "human-solvable", "metric": "pass@1", "name": "BioMysteryBench", "score": 88.8, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "with tools" }, { "dataset": "human-difficult", "metric": "pass@1", "name": "BioMysteryBench", "score": 56.5, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "with tools" }, { "dataset": "11 subtasks", "metric": "macro-average", "name": "LAB-Bench", "score": 86.2, "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "variant": "with tools", "version": "2" }, { "dataset": "642-task public split", "effort": "High", "harness": "mini-swe-agent", "metric": "resolve rate", "name": "SWE-bench Pro", "score": 94.86, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "reported ±1.46; web tools disabled; network-locked protocol; uncapped cost; 250-turn limit", "version": "V2" }, { "cost": 8.64749, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 19.697, "source": "https://www.vals.ai/models/google_gemini-3.8-flash", "variant": "avg@3; reported ±2.31 standard error; no step or cost limit; 8-hour task timeout", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "share of tasks passed (pass@1)", "name": "Terminal-Bench-Science", "score": 10.0, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-gemini-3-5-flash-lite", "variant": "average of three repeats per task", "version": "0.1.0" }, { "dataset": "70 upstream tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 8.57, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single run per model; infrastructure or provider errors retried only", "version": "0.1.0" }, { "effort": "Medium", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 29.7, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "version": "1.0.6" }, { "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 29.7, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "version": "1.0.6" }, { "dataset": "100 expert chart tasks", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 40.9, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 48, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-hy3" }, { "cost": 7.766006178152514, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 30.8468119957291, "source": "https://artificialanalysis.ai/models/gemini-3-8-flash", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1436, "source": "https://artificialanalysis.ai/models/comparisons/gemini-3-8-flash-vs-hy3", "variant": "shell and web browsing; blind pairwise comparisons", "version": "2.1" }, { "cost": 5.025617, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "six private and two public benchmarks", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 55.951, "source": "https://www.vals.ai/models/google_gemini-3.8-flash", "variant": "reported ±1.14; Google provider; default temperature, top-p, and top-k; 65,536 max output tokens", "version": "2.2" }, { "cost": 6.258586089480771, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 30.078569475636, "source": "https://artificialanalysis.ai/models/gemini-3-8-flash-medium", "version": "5.0.1" }, { "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 22.548302237854, "source": "https://artificialanalysis.ai/models/gemini-3-8-flash-low", "version": "5.0.1" }, { "cost": 2.362349413758389, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.8255033557047, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 447 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 1.9671024789823006, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 71.01769911504425, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "High", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 33.1, "source": "https://artificialanalysis.ai/models/gemini-3-8-flash", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Google's most intelligent Flash model, engineered for long-horizon software engineering, autonomous agents, and complex enterprise workflows", "family": "gemini-flash", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 3.8 Flash", "open_weights": false, "release_date": "2026-09-02", "schema": 2 }, "google/gemini-3.8-flash-lite-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Fast, cost-efficient text-to-speech model for high-volume narration and voice agent pipelines", "family": "gemini-flash-lite", "limit": { "context": 8192, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/l16", "audio/mulaw", "audio/alaw" ] }, "name": "Gemini 3.8 Flash Lite TTS", "open_weights": false, "release_date": "2026-09-22", "schema": 2, "type": "speech", "v2_only": true }, "google/gemini-3.8-flash-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Flagship text-to-speech model for expressive, studio-grade voices and long-form multi-speaker audio", "family": "gemini-flash", "limit": { "context": 8192, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/l16", "audio/mulaw", "audio/alaw" ] }, "name": "Gemini 3.8 Flash TTS", "open_weights": false, "release_date": "2026-09-22", "schema": 2, "type": "speech", "v2_only": true }, "google/gemini-3.8-live": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Low-latency Live API model for real-time voice agents with interleaved reasoning and async tools", "family": "gemini-flash", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video", "audio/pcm" ], "output": [ "text", "audio/pcm" ] }, "name": "Gemini 3.8 Live", "open_weights": false, "release_date": "2026-09-15", "schema": 2, "type": "realtime", "v2_only": true }, "google/gemini-3.8-live-extended-thinking": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Live API voice model with configurable background reasoning for multi-step real-time agents", "family": "gemini-flash", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video", "audio/pcm" ], "output": [ "text", "audio/pcm" ] }, "name": "Gemini 3.8 Live Extended Thinking", "open_weights": false, "release_date": "2026-09-15", "schema": 2, "type": "realtime", "v2_only": true }, "google/gemini-4-argon": { "benchmarks": [ { "dataset": "DeepSWE v1.1 (113 tasks)", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 77.9, "source": "https://deepmind.google/models/model-cards/gemini-4-argon/", "version": "1.1" }, { "dataset": "SWE-bench Pro V2 Full (642 tasks); reported 99.22 ± 0.45%", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 99.22, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "Full", "version": "2.0.0" }, { "cost": 17.63994, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0 (66 tasks); reported ±2.31 percentage points", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 57.576, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1", "version": "4.0.0" }, { "effort": "High", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 57.4, "source": "https://deepmind.google/models/model-cards/gemini-4-argon/", "variant": "Google self-computed; pass@1; repeat count and harness not specified", "version": "4.0.0" }, { "effort": "High", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 57.6, "source": "https://deepmind.google/models/model-cards/gemini-4-argon/", "variant": "Google self-computed; pass@1; verifier timeout set to 6×; harness not specified", "version": "0.1.0" }, { "dataset": "Terminal-Bench-Science v0.1.0 (70 tasks)", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 53, "source": "https://artificialanalysis.ai/models/comparisons/gemini-4-argon-vs-gpt-6-astra-high", "variant": "pass@1 averaged over 3 repeats", "version": "0.1.0" }, { "dataset": "Terminal-Bench-Science v0.1.0 (70 tasks); reported ±5.98 percentage points", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 44.29, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single pass@1 run; infrastructure or provider errors may be retried", "version": "0.1.0" }, { "dataset": "Zapier 1.0.6 held-out private task set (600+ tasks; 47 apps)", "effort": "High", "harness": "Zapier AutomationBench", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 51.29, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "Zapier 1.0.6 held-out private task set (600+ tasks; 47 apps)", "effort": "Medium", "harness": "Zapier AutomationBench", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 50.08, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "OSWorld-2.1 offline subset (82 tasks)", "effort": "High", "harness": "Gemini CUA", "metric": "partial score", "name": "OSWorld", "score": 74.7, "source": "https://deepmind.google/models/gemini/", "variant": "maximum across 3 single-attempt runs; offline subset", "version": "osworld-v2.1" }, { "effort": "High reasoning", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 71.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "without tools", "version": "unspecified" }, { "cost": 13.372731, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "Vals Index composite (finance, coding, legal, tax)", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 69.237, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 9.946608711550533, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 50.4121935905389, "source": "https://artificialanalysis.ai/models/gemini-4-argon", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "High", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 57.1, "source": "https://artificialanalysis.ai/models/gemini-4-argon", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Google's flagship reasoning model for complex software engineering, deep math and science, and long-horizon autonomous agents", "family": "gemini", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 4 Argon", "open_weights": false, "release_date": "2026-10-13", "schema": 2 }, "google/gemini-4-carbon": { "benchmarks": [ { "cost": 10.508065, "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 70.45, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "version": "2.2" }, { "cost": 7.452059, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 63.131, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "cost": 6.585776867225546, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 58.2587824022788, "source": "https://artificialanalysis.ai/models/gemini-4-carbon-medium", "version": "5.0.1" }, { "cost": 7.755545745286556, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 60.025738829526, "source": "https://artificialanalysis.ai/models/gemini-4-carbon", "version": "5.0.1" }, { "cost": 7.186577262951972, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 54.2206114502924, "source": "https://artificialanalysis.ai/models/gemini-4-carbon-low", "version": "5.0.1" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "High", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 69.1, "source": "https://artificialanalysis.ai/models/gemini-4-carbon", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true } }, "description": "Google reasoning model evaluated by Vals on agentic finance, coding, legal, and tax tasks; model details are currently based on Vals' profile.", "family": "gemini", "limit": {}, "modalities": { "input": [ "text", "image", "video" ], "output": [ "text" ] }, "name": "Gemini 4 Carbon", "open_weights": false, "release_date": "2026-10-28", "schema": 2 }, "google/gemini-4-hydrogen": { "benchmarks": [ { "effort": "xhigh", "harness": "Antigravity-CLI (agy) 1.4.8", "metric": "subtask-weighted reward", "name": "SWE-sweep", "score": 13.5, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; pass@1; official integration; safety filters enabled" }, { "effort": "high", "harness": "mini-swe-agent vision", "metric": "cheating-adjusted accuracy", "name": "Terminal-Bench", "score": 57.0, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; Average Reward over 5 attempts per task", "version": "4.1" }, { "dataset": "122 tasks; October 17, 2026 task version, without the October 26 update", "effort": "xhigh", "harness": "Antigravity-CLI (agy) 1.4.8", "metric": "averaged rubric Score", "name": "Agents' Last Exam", "score": 71.6, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; pass@1; 5-hour window; official integration; safety-filtered responses become empty strings, but episodes may continue", "version": "1.1" }, { "metric": "%", "name": "AutomationBench-AA", "score": 76.1, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Reported by Google from the Artificial Analysis leaderboard" }, { "metric": "bank balance (USD)", "name": "Vending-Bench", "score": 14549, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Reported by Google from the Andon Labs public leaderboard", "version": "2" }, { "dataset": "OSWorld-2.1 offline subset (82 tasks)", "effort": "xhigh", "harness": "Gemini CUA", "metric": "partial score", "name": "OSWorld", "score": 77.6, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; maximum across 3 single-attempt runs; official Docker and evaluator; 1080p; 500-step limit; screenshot-only observations; PyAutoGUI; parallel batch tool calling; all screenshots retained with compaction; safety-filtered responses become empty strings, but episodes may continue", "version": "osworld-v2.1" }, { "effort": "xhigh", "metric": "Climb WSR accuracy", "name": "MKA", "score": 53.4, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; pass@1" }, { "effort": "xhigh", "metric": "pass@1", "name": "Chartography", "score": 66.6, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; without tools; following the official Surge public leaderboard" }, { "effort": "xhigh", "harness": "mini-swe-agent vision", "metric": "Average Reward", "name": "Terminal-Bench-Science", "score": 51.4, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Google self-computed; averaged over 5 attempts per task", "version": "0.1" }, { "metric": "%", "name": "HLE-Diamond", "score": 58.8, "source": "https://deepmind.google/models/model-cards/gemini-4-hydrogen/", "variant": "Reported by Google from the Artificial Analysis leaderboard" }, { "cost": 9.838059, "cost_unit": "USD/test", "effort": "xhigh", "metric": "performance score", "name": "Vals Index", "score": 68.492, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.BQOv7o-i.json", "version": "2.2" }, { "cost": 3.799925, "cost_unit": "USD/test", "effort": "xhigh", "metric": "accuracy", "name": "Vals Finance Index", "score": 66.136, "source": "https://www.vals.ai/_astro/benchmark_view_finance_index.DyW6srHB.json", "version": "1" }, { "cost": 4.47377, "cost_unit": "USD/test", "effort": "xhigh", "metric": "accuracy", "name": "Vals Legal Index", "score": 73.629, "source": "https://www.vals.ai/_astro/benchmark_view_legal_index.D3brxS62.json", "version": "1" }, { "cost": 3.170206, "cost_unit": "USD/test", "effort": "xhigh", "metric": "partial-credit score", "name": "EMB v1.1", "score": 75.798, "source": "https://www.vals.ai/_astro/benchmark_view_emb_v1_1.BClPXHvL.json", "version": "1.1" }, { "cost": 4.358646, "cost_unit": "USD/test", "effort": "xhigh", "metric": "Partial Credit", "name": "Finance Agent v2.1", "score": 65.118, "source": "https://www.vals.ai/_astro/benchmark_view_finance_agent_v2_1.vQvzZRw-.json", "variant": "dealbreaker-gated, severity-weighted per-check scores; 2-hour task limit", "version": "2.1" }, { "cost": 6.268559, "cost_unit": "USD/test", "effort": "xhigh", "metric": "All-Pass", "name": "Legal Research Bench", "score": 53.846, "source": "https://www.vals.ai/_astro/benchmark_view_legal_research.BjgV0FJw.json", "variant": "3-hour task limit; GPT 5.4 rubric judge", "version": "1" }, { "cost": 3.870924, "cost_unit": "USD/test", "dataset": "193-question private test set", "effort": "xhigh", "metric": "citation-adjusted all-pass accuracy", "name": "Tax Agent Bench", "score": 52.305, "source": "https://www.vals.ai/_astro/benchmark_view_tax_agent_bench.CQdap7eo.json", "variant": "all-pass multiplied by (0.7 + 0.3 times citation quality); 3-hour task limit; GPT 5.4 rubric judge", "version": "1" }, { "cost": 4.553735, "cost_unit": "USD/test", "effort": "xhigh", "harness": "OpenHands", "metric": "score", "name": "Vibe Code Bench v1.1", "score": 87.777, "source": "https://www.vals.ai/_astro/benchmark_view_vibe_code_bench.DLGw-pgN.json", "version": "1.1" }, { "cost": 3.281826, "cost_unit": "USD/test", "dataset": "120-task private holdout", "effort": "xhigh", "metric": "Task Pass Rate", "name": "HLAB-Vals-1.1", "score": 11.25, "source": "https://www.vals.ai/_astro/benchmark_view_legal_agent_benchmark.D_vTXw_F.json", "variant": "internet disabled; mean of GPT 5.5 and Claude Sonnet 4.6 task pass rates; task passes only if all criteria pass", "version": "1.1" }, { "cost": 7.562172, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 54.04, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.B2-ATImw.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0" }, { "cost": 0.2922227000200506, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 26.8936694361562, "source": "https://artificialanalysis.ai/models/gemini-4-hydrogen-low", "version": "5.0.1" }, { "cost": 3.1601590757006157, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 53.9192287317542, "source": "https://artificialanalysis.ai/models/gemini-4-hydrogen", "version": "5.0.1" }, { "cost": 1.8733140062520008, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 50.7870826951304, "source": "https://artificialanalysis.ai/models/gemini-4-hydrogen-high", "version": "5.0.1" }, { "cost": 0.7979866055252406, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 41.2329566992579, "source": "https://artificialanalysis.ai/models/gemini-4-hydrogen-medium", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Google's Gemini 4 model for everyday coding and agentic workflows, with four thinking levels and a 1M-token context window", "family": "gemini", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini 4 Hydrogen", "open_weights": false, "release_date": "2026-11-12", "schema": 2 }, "google/gemini-decision-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Google's decision model that returns calibrated probabilities for typed yes/no, choice and score questions about text, images, audio, video and PDFs", "family": "gemini", "limit": { "context": 262144, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "image/gif", "image/bmp", "image/tiff", "audio/wav", "audio/mp3", "audio/mpeg", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/m4a", "audio/opus", "audio/webm", "audio/l16", "audio/alaw", "audio/mulaw", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/webm", "video/wmv", "video/x-flv", "video/3gpp", "application/pdf" ], "output": [ "decision" ] }, "name": "Gemini Decision 1", "open_weights": false, "release_date": "2026-11-11", "schema": 2, "type": "decision", "v2_only": true }, "google/gemini-embedding-001": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Embedding model for semantic search, retrieval, clustering, and ranking pipelines", "family": "gemini", "knowledge": "2025-05", "limit": { "context": 2048, "output": 1 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Gemini Embedding 001", "open_weights": false, "release_date": "2025-05-20", "schema": 2, "type": "embedding" }, "google/gemini-embedding-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multimodal embedding model mapping text, images, video, audio, and PDFs into a unified embedding space", "family": "gemini", "knowledge": "2025-11", "limit": { "context": 8192, "output": 3072 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/mp3", "audio/wav", "video/mp4", "video/mov", "application/pdf" ], "output": [ "embedding" ] }, "name": "Gemini Embedding 2", "open_weights": false, "release_date": "2026-04-22", "schema": 2, "type": "embedding" }, "google/gemini-embedding-2-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Preview of Gemini Embedding 2, a multimodal embedding model for text, images, video, audio, and PDFs", "family": "gemini", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/mp3", "audio/wav", "video/mp4", "video/mov", "application/pdf" ], "output": [ "embedding" ] }, "name": "Gemini Embedding 2 Preview", "open_weights": false, "release_date": "2026-03-10", "schema": 2, "type": "embedding", "v2_only": true }, "google/gemini-flash-latest": { "benchmarks": [ { "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.3, "source": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "variant": "self-computed; LiteLLM 1.96; high thinking", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Medium", "metric": "pass@1", "name": "DeepSWE", "score": 65, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "official v1.1 leaderboard; uncertainty ±3 percentage points", "version": "1.1" }, { "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench", "score": 12.12, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1; 3 runs; ±1 standard error", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 7.14, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single-run pass@1; infrastructure/provider errors retried", "version": "0.1.0" }, { "dataset": "private set", "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 30.4, "source": "https://zapier.com/benchmarks/claude-haiku5_5-report", "version": "1.0.6" }, { "dataset": "100-task full task set; 10 runs per task; Gemini 3.5 Flash judge", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 40.4, "source": "https://surgehq.ai/benchmarks/chartography" }, { "dataset": "2,158 text-only questions from the original HLE set", "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 48, "source": "https://artificialanalysis.ai/models/comparisons/claude-fable-5-1-high-vs-gemini-3-7-flash" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "High-efficiency Gemini model for agentic workflows, coding, and multimodal reasoning", "family": "gemini-flash", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini Flash Latest", "open_weights": false, "release_date": "2026-08-13", "schema": 2 }, "google/gemini-flash-lite-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Gemini model balancing multimodal reasoning, tool use, and cost", "family": "gemini-flash-lite", "knowledge": "2026-03", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm", "application/pdf" ], "output": [ "text" ] }, "name": "Gemini Flash-Lite Latest", "open_weights": false, "release_date": "2026-07-21", "schema": 2 }, "google/gemini-nano-banana-2.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, multi-turn editing, and visual design workflows", "family": "gemini-flash", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text", "image/png", "image/jpeg" ] }, "name": "Nano Banana 2.1", "open_weights": false, "release_date": "2026-10-06", "schema": 2, "type": "image" }, "google/gemini-omni-1.1-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video generation and editing model for scene extension, frame interpolation, and reference-guided multimodal control", "family": "gemini", "limit": { "context": 1048576, "output": 57920 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "video/mp4" ] }, "name": "Gemini Omni 1.1 Flash", "open_weights": false, "release_date": "2026-08-27", "schema": 2, "type": "video" }, "google/gemini-omni-1.2-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video generation and editing model with native audio, up to 15-second clips, extension, frame interpolation, and reference-guided multimodal control", "family": "gemini", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "video/mp4" ] }, "name": "Gemini Omni 1.2 Flash", "open_weights": false, "release_date": "2026-11-05", "schema": 2, "type": "video", "v2_only": true }, "google/gemini-omni-flash-preview": { "benchmarks": [ { "date": "2026-06-30", "metric": "Elo", "name": "LMArena Text-to-Video Arena", "score": 1527, "source": "https://venturebeat.com/technology/googles-gemini-omni-flash-hits-the-api-turning-enterprise-video-production-into-a-conversation" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video generation and editing model for fast, conversational text- and image-to-video workflows", "family": "gemini", "limit": { "context": 1048576, "output": 57920 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "video/mp4" ] }, "name": "Gemini Omni Flash Preview", "open_weights": false, "release_date": "2026-06-30", "schema": 2, "type": "video" }, "google/gemini-robotics-er-1.6-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Vision-language model for embodied reasoning: spatial understanding, task planning, and physical-world agentic robotics", "family": "gemini", "knowledge": "2025-01", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm" ], "output": [ "text" ] }, "name": "Gemini Robotics-ER 1.6 Preview", "open_weights": false, "release_date": "2026-04-14", "schema": 2 }, "google/gemini-robotics-er-2-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Embodied reasoning model for robotics: spatial understanding, video progress, and multi-robot orchestration", "family": "gemini", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp", "audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac", "audio/mpeg", "audio/m4a", "audio/l16", "audio/opus", "audio/alaw", "audio/mulaw", "audio/webm" ], "output": [ "text" ] }, "name": "Gemini Robotics-ER 2 Preview", "open_weights": false, "release_date": "2026-07-30", "schema": 2, "v2_only": true }, "google/gemini-robotics-er-2-streaming-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Live API robotics model for low-latency robot agents over continuous audio and video", "family": "gemini", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/jpeg", "video", "audio/pcm" ], "output": [ "text" ] }, "name": "Gemini Robotics-ER 2 Streaming Preview", "open_weights": false, "release_date": "2026-07-30", "schema": 2, "type": "realtime", "v2_only": true }, "google/gemma-2-27b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Gemma instruction model for efficient chat and self-hosted deployments", "family": "gemma", "knowledge": "2024-06-30", "limit": { "context": 8192, "output": 2048 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/google/gemma-2-27b-it" ], "total_billion": 27.22712832 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Gemma 2 27B", "open_weights": true, "release_date": "2024-07-13", "schema": 2 }, "google/gemma-2-2b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Small open Gemma 2 instruction model for lightweight chat and on-device use", "family": "gemma", "license": "Gemma Terms of Use", "limit": { "context": 8192, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint, not the nominal 2B designation.", "sources": [ "https://huggingface.co/api/models/google/gemma-2-2b-it" ], "total_billion": 2.614341888 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Gemma 2 2B IT", "open_weights": true, "release_date": "2024-07-31", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-2-2b-it" } ] }, "google/gemma-3-12b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal Gemma instruction model for multilingual text generation and image understanding", "family": "gemma", "knowledge": "2024-08", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint.", "sources": [ "https://huggingface.co/api/models/google/gemma-3-12b-it" ], "total_billion": 12.18732504 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemma 3 12B IT", "open_weights": true, "release_date": "2025-03-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-3-12b-it" } ] }, "google/gemma-3-27b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Largest open Gemma 3 instruction model for multilingual text generation and visual understanding", "family": "gemma", "knowledge": "2024-08", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint.", "sources": [ "https://huggingface.co/api/models/google/gemma-3-27b-it" ], "total_billion": 27.43240664 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemma 3 27B IT", "open_weights": true, "release_date": "2025-03-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-3-27b-it" } ] }, "google/gemma-3-4b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal Gemma instruction model for efficient text generation and image understanding", "family": "gemma", "knowledge": "2024-08", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint.", "sources": [ "https://huggingface.co/api/models/google/gemma-3-4b-it" ], "total_billion": 4.300079472 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemma 3 4B IT", "open_weights": true, "release_date": "2025-03-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-3-4b-it" } ] }, "google/gemma-3n-e2b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Efficient on-device Gemma 3n model for text, image, and audio understanding", "family": "gemma", "knowledge": "2024-06", "license": "Gemma Terms of Use", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official card's raw parameter count, not the effective 2B memory-footprint designation. Effective parameters are not treated as per-token active parameters.", "sources": [ "https://huggingface.co/google/gemma-3n-E2B-it" ], "total_billion": 6 } } }, "modalities": { "input": [ "text", "image", "audio" ], "output": [ "text" ] }, "name": "Gemma 3n E2B IT", "open_weights": true, "release_date": "2025-06-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-3n-E2B-it" } ] }, "google/gemma-3n-e4b-it": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Efficient on-device Gemma 3n model for text, image, and audio understanding", "family": "gemma", "knowledge": "2024-06", "license": "Gemma Terms of Use", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official card's raw parameter count, not the effective 4B memory-footprint designation. Effective parameters are not treated as per-token active parameters.", "sources": [ "https://huggingface.co/google/gemma-3n-E4B-it" ], "total_billion": 8 } } }, "modalities": { "input": [ "text", "image", "audio" ], "output": [ "text" ] }, "name": "Gemma 3n E4B IT", "open_weights": true, "release_date": "2025-06-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-3n-E4B-it" } ] }, "google/gemma-4-12b-it": { "benchmarks": [ { "name": "Humanity's Last Exam", "score": 5.2, "source": "https://ai.google.dev/gemma/docs/core/model_card_4", "variant": "no tools" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Gemma 4 instruction model for open, self-hosted chat and reasoning", "family": "gemma", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "12B Unified column of the official total-parameter table; encoder-free multimodal architecture.", "sources": [ "https://huggingface.co/google/gemma-4-12B-it" ], "total_billion": 11.95 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Gemma 4 12B IT", "open_weights": true, "release_date": "2026-06-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-4-12B-it" } ] }, "google/gemma-4-26b-a4b-it": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.7, "source": "https://ai.google.dev/gemma/docs/core/model_card_4", "variant": "instruction-tuned; HLE no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 17.2, "source": "https://ai.google.dev/gemma/docs/core/model_card_4", "variant": "instruction-tuned; HLE with search" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Gemma instruction model for efficient chat and self-hosted deployments", "family": "gemma", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.8, "notes": "Total is the Hub-published BF16 count for the complete multimodal checkpoint, not the table's 25.2B language component. Per-token active count is explicitly reported in the official model card.", "sources": [ "https://huggingface.co/api/models/google/gemma-4-26B-A4B-it", "https://huggingface.co/google/gemma-4-26B-A4B-it" ], "total_billion": 25.805936206 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemma 4 26B A4B IT", "open_weights": true, "release_date": "2026-04-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-4-26B-A4B-it" } ] }, "google/gemma-4-31b-it": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 19.5, "source": "https://ai.google.dev/gemma/docs/core/model_card_4", "variant": "instruction-tuned; HLE no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 26.5, "source": "https://ai.google.dev/gemma/docs/core/model_card_4", "variant": "instruction-tuned; HLE with search" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 4.38438284215374, "source": "https://artificialanalysis.ai/models/gemma-4-31b", "variant": "Reasoning", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Largest Gemma 4 instruction model for open, self-hosted chat and reasoning", "family": "gemma", "knowledge": "2025-01", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not the table's 30.7B language component.", "sources": [ "https://huggingface.co/api/models/google/gemma-4-31B-it" ], "total_billion": 31.273088876 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text" ] }, "name": "Gemma 4 31B IT", "open_weights": true, "release_date": "2026-04-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-4-31B-it" } ] }, "google/gemma-4-E2B-it": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Gemma instruction model for efficient chat and self-hosted deployments", "family": "gemma", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official total with embeddings, not the 2.3B effective count. Vision and audio encoders are listed separately; no component sum inferred.", "sources": [ "https://huggingface.co/google/gemma-4-E2B-it" ], "total_billion": 5.1 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Gemma 4 E2B IT", "open_weights": true, "release_date": "2026-04-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-4-E2B-it" } ] }, "google/gemma-4-E4B-it": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Gemma instruction model for efficient chat and self-hosted deployments", "family": "gemma", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official total with embeddings, not the 4.5B effective count. Vision and audio encoders are listed separately; no component sum inferred.", "sources": [ "https://huggingface.co/google/gemma-4-E4B-it" ], "total_billion": 8 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Gemma 4 E4B IT", "open_weights": true, "release_date": "2026-04-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/google/gemma-4-E4B-it" } ] }, "google/lyria-3-clip-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Music generation model for short 30-second clips, loops, and previews from text or image prompts", "family": "lyria", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "audio/mp3" ] }, "name": "Lyria 3 Clip Preview", "open_weights": false, "release_date": "2026-03-25", "schema": 2, "type": "audio" }, "google/lyria-3-pro-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Music generation model for full-length songs from text or images with vocals and structure", "family": "lyria", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "audio/mp3" ] }, "name": "Lyria 3 Pro Preview", "open_weights": false, "release_date": "2026-03-25", "schema": 2, "type": "audio" }, "google/lyria-3.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Music generation model for full-length songs with vocals and structure from text or image prompts", "family": "lyria", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif" ], "output": [ "text", "audio/mp3", "audio/wav" ] }, "name": "Lyria 3.5", "open_weights": false, "release_date": "2026-09-03", "schema": 2, "type": "audio", "v2_only": true }, "google/lyria-realtime-exp": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": false }, "description": "Experimental streaming music generation model steered in real time by weighted text prompts", "family": "lyria", "limit": { "context": 1048576, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "audio/pcm" ] }, "name": "Lyria RealTime", "open_weights": false, "release_date": "2025-05-20", "schema": 2, "type": "realtime", "v2_only": true }, "google/text-embedding-005": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Gecko-based text embedding model specialized in English and code tasks, producing up to 768-dimensional vectors", "family": "text-embedding", "limit": { "context": 2048, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Text Embedding 005", "open_weights": false, "release_date": "2024-11-18", "schema": 2, "type": "embedding", "v2_only": true }, "google/text-multilingual-embedding-002": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Gecko-based multilingual text embedding model producing up to 768-dimensional vectors", "family": "text-embedding", "limit": { "context": 2048, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Text Multilingual Embedding 002", "open_weights": false, "release_date": "2024-05-14", "schema": 2, "type": "embedding", "v2_only": true }, "google/veo-3.1-fast-generate-001": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1 Fast", "open_weights": false, "release_date": "2025-11-17", "schema": 2, "type": "video", "v2_only": true }, "google/veo-3.1-fast-generate-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "last_updated": "2026-01-01", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1 Fast Preview", "open_weights": false, "release_date": "2025-10-15", "schema": 2, "type": "video" }, "google/veo-3.1-generate-001": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1", "open_weights": false, "release_date": "2025-11-17", "schema": 2, "type": "video", "v2_only": true }, "google/veo-3.1-generate-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "last_updated": "2026-01", "limit": { "context": 1024, "output": 1 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1 Preview", "open_weights": false, "release_date": "2025-10-15", "schema": 2, "type": "video" }, "google/veo-3.1-lite-generate-001": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1 Lite", "open_weights": false, "release_date": "2026-04-02", "schema": 2, "type": "video", "v2_only": true }, "google/veo-3.1-lite-generate-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for prompt-guided generation, editing, and motion workflows", "family": "veo", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Veo 3.1 Lite Preview", "open_weights": false, "release_date": "2026-03-31", "schema": 2, "type": "video" }, "gryphe/mythomax-l2-13b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Llama 2 merge combining MythoLogic and Huginn for creative writing and roleplay", "limit": { "context": 4096, "output": 3200 }, "metadata": { "modelatlas": { "parameters": { "notes": "13,015,864,320 F16 weight values independently counted from ZIP data.pkl tensor descriptors across all thirteen indexed shards. All 363 tensor names match the shard index; no weight payloads were loaded. This is not inferred from the name or file sizes.", "sources": [ "https://huggingface.co/Gryphe/MythoMax-L2-13b/blob/58e77dd48a65176f97f6f376c93efe9caad9c130/pytorch_model.bin.index.json" ], "total_billion": 13.01586432 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MythoMax L2 13B", "open_weights": true, "release_date": "2023-08-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Gryphe/MythoMax-L2-13b" } ] }, "hexgrad/kokoro-82m": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lightweight open text-to-speech model", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/hexgrad/Kokoro-82M" ], "total_billion": 0.082 } } }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "Kokoro 82M", "open_weights": true, "release_date": "2024-12", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/hexgrad/Kokoro-82M" } ] }, "heygen/avatar-iv": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image-to-video avatar model that turns one photo and a script or audio track into a lip-synced talking video", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "Avatar IV", "open_weights": false, "release_date": "2025-05", "schema": 2, "type": "video", "v2_only": true }, "heygen/heygen-video-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose video model for text, image and reference to video with generated dialogue, ambience and effects", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "video/mp4", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "HeyGen Video", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "video", "v2_only": true }, "heygen/heygen-voice-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-speech model with default voices and HeyGen voice clones", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "HeyGen Voice", "open_weights": false, "release_date": "2026-10-13", "schema": 2, "type": "speech", "v2_only": true }, "heygen/heygen-voice-1-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lower-latency version of HeyGen Voice", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "HeyGen Voice Turbo", "open_weights": false, "release_date": "2026-10-13", "schema": 2, "type": "speech", "v2_only": true }, "ibm/granite-4-h-micro": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact open-weight hybrid Granite model for lightweight enterprise chat and tool calling", "family": "granite", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/ibm-granite/granite-4.0-h-micro" ], "total_billion": 3 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Granite-4.0-H-Micro", "open_weights": true, "release_date": "2025-10-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-micro" } ] }, "ibm/granite-4-h-small": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight hybrid model for enterprise chat, coding, retrieval-augmented generation, and tool-calling workloads", "family": "granite", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/ibm-granite/granite-4.0-h-small" ], "total_billion": 32 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Granite-4.0-H-Small", "open_weights": true, "release_date": "2025-10-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ibm-granite/granite-4.0-h-small" } ] }, "ibm/granite-4.2-8b": { "benchmarks": [ { "cost": 0.0404628507686443, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 5.09826495217051, "source": "https://artificialanalysis.ai/models/granite-4-2-8b/", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "family": "granite", "limit": { "context": 131072, "output": 117964 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/ibm-granite/granite-4.2-8b" ], "total_billion": 8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Granite 4.2 8B", "open_weights": true, "release_date": "2026-08-25", "schema": 2 }, "inception/mercury-2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "family": "mercury", "limit": { "context": 128000, "output": 50000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mercury 2", "open_weights": false, "release_date": "2026-02-24", "schema": 2 }, "inception/mercury-2.5": { "benchmarks": [ { "cost": 0.368017, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "family": "mercury", "limit": { "context": 260000, "output": 65536 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mercury 2.5", "open_weights": false, "release_date": "2026-09-08", "schema": 2 }, "inception/mercury-3": { "benchmarks": [ { "cost": 0.421803, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0 (66 tasks), Vals AI evaluation", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 3.03, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; mean of 3 full pass@1 runs; 8-hour agent time limit per task", "version": "4.0.0" }, { "cost": 0.477863, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 43.659, "source": "https://www.vals.ai/models/inception_mercury-3", "version": "2.2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Inception's diffusion reasoning model for chat, multi-step problem solving, and tool use", "family": "mercury", "last_updated": "2026-10-20", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mercury 3", "open_weights": false, "release_date": "2026-10-16", "schema": 2 }, "inception/mercury-decide": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Inception's diffusion decision model that returns probabilities for typed yes/no, choice and score questions about text or images", "family": "mercury", "limit": { "context": 65536, "output": 50000 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "decision" ] }, "name": "Mercury Decide", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "decision", "v2_only": true }, "inception/mercury-edit-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Code editing dLLM for autocomplete (FIM) and next-edit suggestions", "family": "mercury", "limit": { "context": 32000, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mercury Edit 2", "open_weights": false, "release_date": "2026-03-30", "schema": 2 }, "inclusionai/ling-3.0-flash": { "benchmarks": [ { "dataset": "ScaleAI/SWE-bench_Pro", "harness": "OpenHands", "name": "SWE-bench Pro", "score": 56.6, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/main/.eval_results/swe-bench_pro.yaml" }, { "dataset": "cais/hle", "name": "Humanity's Last Exam", "score": 22.7, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/main/.eval_results/hle.yaml" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Hybrid-linear MoE reasoning model with 124B total and 5.1B active parameters for efficient agents", "family": "ling", "license": "MIT", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 5.1, "sources": [ "https://huggingface.co/inclusionAI/Ling-3.0-flash" ], "total_billion": 124 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ling 3.0 Flash", "open_weights": true, "release_date": "2026-07-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash" } ] }, "inclusionai/ling-3.0-flash-fin": { "benchmarks": [ { "harness": "ReAct with web search, visit, and Python", "metric": "score", "name": "FinFIRST", "score": 52.85, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort" }, { "dataset": "expert-validated internal set", "harness": "ReAct with web search, visit, and Python", "metric": "score", "name": "FinSearchComp Verified", "score": 77.04, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort" }, { "dataset": "internal", "metric": "score", "name": "FinCRAFT", "score": 51.74, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort" }, { "metric": "strict-pass", "name": "Finance Agent", "score": 69.19, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort; 10 runs", "version": "1.1" }, { "metric": "strict-pass", "name": "Finance Agent", "score": 59.81, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort; 20 runs", "version": "2" }, { "metric": "score", "name": "APEX-Agents", "score": 29.17, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 0, "source": "https://artificialanalysis.ai/zh/articles/ant-group-releases-finance-focused-ling-3-0-flash-fin", "variant": "3 repeats per task", "version": "4.0.0" }, { "harness": "Claude Code 2.1.173", "metric": "score", "name": "SpreadsheetBench", "score": 86.5, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort; temperature=0.6; 120 turns; 3h timeout", "version": "1" }, { "harness": "Claude Code 2.1.173", "metric": "score", "name": "SpreadsheetBench", "score": 21.81, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort; temperature=0.6; 300 turns; 3h timeout", "version": "2" }, { "metric": "score", "name": "tau3-bench", "score": 41, "source": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin", "variant": "highest reasoning effort; banking" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Finance-enhanced model for financial research, multi-step investment workflows, and long-horizon planning and execution", "family": "ling", "last_updated": "2026-09-03", "license": "MIT", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 5.1, "sources": [ "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin" ], "total_billion": 124 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ling 3.0 Flash Fin", "open_weights": true, "release_date": "2026-08-27", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash-Fin" } ] }, "inclusionai/ling-3.0-flash-sante": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Health and medicine specialization of Ling 3.0 Flash for medical reasoning and evidence-based retrieval", "family": "ling", "limit": { "context": 262144, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ling 3.0 Flash Sante", "open_weights": false, "release_date": "2026-09-04", "schema": 2 }, "inclusionai/ling-3.0-flash-vl": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Native multimodal hybrid MoE model for image and video understanding, reasoning, and visual agents", "family": "ling", "license": "MIT", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 5.5, "sources": [ "https://huggingface.co/inclusionAI/Ling-3.0-flash-VL" ], "total_billion": 124 } } }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "text" ] }, "name": "Ling 3.0 Flash VL", "open_weights": true, "release_date": "2026-09-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/inclusionAI/Ling-3.0-flash-VL" } ] }, "inclusionai/ling-3.0-tiny": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lightweight hybrid-linear MoE reasoning model with 7.9B total and 1.3B active parameters for on-device deployment and offline agents", "family": "ling", "license": "MIT", "limit": { "context": 131072, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ling 3.0 Tiny", "open_weights": true, "release_date": "2026-08-07", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/inclusionAI/Ling-3.0-tiny" } ] }, "inclusionai/ling-3.1-flash": { "benchmarks": [ { "dataset": "harborframework/terminal-bench", "harness": "LingCode", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 46.03, "source": "https://huggingface.co/inclusionAI/Ling-3.1-flash/discussions/6/files", "version": "4.0.0" }, { "dataset": "Artificial Analysis Terminal-Bench 4.0 (66 tasks)", "harness": "mini-swe-agent (pass@1 averaged over three repeats)", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 33, "source": "https://artificialanalysis.ai/models/comparisons/reflection-beam-vs-ling-3-1-flash", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.5, "source": "https://artificialanalysis.ai/models/ling-3-1-flash-1019", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 17.2, "source": "https://artificialanalysis.ai/models/ling-3-1-flash-1019", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "ling", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 25, "sources": [ "https://huggingface.co/inclusionAI/Ling-3.1-flash" ], "total_billion": 560 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ling 3.1 Flash", "open_weights": true, "release_date": "2026-09-29", "schema": 2 }, "inclusionai/ling-3.2-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Native multimodal MoE model with 560B total and 25B active parameters for complex visual understanding, multimodal agents, 3D interaction, and medical imaging", "family": "ling", "limit": { "context": 1000000, "output": 32768 }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "text" ] }, "name": "Ling 3.2 Flash", "open_weights": false, "release_date": "2026-11-05", "schema": 2 }, "inclusionai/ming-image-0.1-design": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Design image generation and editing model for UI drafts and visual design workflows", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image" ], "output": [ "image" ] }, "name": "Ming Image 0.1 Design", "open_weights": false, "release_date": "2026-10-27", "schema": 2, "type": "image", "v2_only": true }, "inclusionai/ming-image-0.1-design-layer": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model that decomposes a design into separate transparent layers for editing workflows", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image" ], "output": [ "image/png" ] }, "name": "Ming Image 0.1 Design Layer", "open_weights": false, "release_date": "2026-10-27", "schema": 2, "type": "image", "v2_only": true }, "inference-net/schematron-v2-small": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Schematron V2 Small", "open_weights": false, "release_date": "2026-04-16", "schema": 2 }, "inference-net/schematron-v2-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "limit": { "context": 128000, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Schematron V2 Turbo", "open_weights": false, "release_date": "2026-04-16", "schema": 2 }, "intfloat/e5-base-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model for retrieval and semantic similarity", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes the separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/intfloat/e5-base-v2" ], "total_billion": 0.10948224 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "E5 Base v2", "open_weights": true, "release_date": "2023-05", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/intfloat/e5-base-v2" } ] }, "intfloat/e5-large-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model for retrieval and semantic similarity", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes the separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/intfloat/e5-large-v2" ], "total_billion": 0.335141888 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "E5 Large v2", "open_weights": true, "release_date": "2023-05", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/intfloat/e5-large-v2" } ] }, "intfloat/multilingual-e5-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text embedding model for retrieval and semantic similarity", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes the separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/intfloat/multilingual-e5-large" ], "total_billion": 0.559890432 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Multilingual E5 Large", "open_weights": true, "release_date": "2023-06", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/intfloat/multilingual-e5-large" } ] }, "intfloat/multilingual-e5-large-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Instruction-tuned multilingual text embedding model for retrieval and semantic similarity", "license": "MIT", "limit": { "context": 512, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Multilingual E5 Large Instruct", "open_weights": true, "release_date": "2024-02-08", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/intfloat/multilingual-e5-large-instruct" } ] }, "inworld/inworld-tts-1.5-max": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Realtime text-to-speech model with expressive, stable delivery and voice cloning across 15 languages", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/ogg", "audio/alaw", "audio/basic", "audio/flac", "audio/pcm" ] }, "name": "TTS-1.5 Max", "open_weights": false, "release_date": "2026-01-21", "schema": 2, "type": "speech", "v2_only": true }, "inworld/inworld-tts-1.5-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lowest-latency realtime text-to-speech model of the TTS-1.5 family, with voice cloning across 15 languages", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/ogg", "audio/alaw", "audio/basic", "audio/flac", "audio/pcm" ] }, "name": "TTS-1.5 Mini", "open_weights": false, "release_date": "2026-01-21", "schema": 2, "type": "speech", "v2_only": true }, "inworld/inworld-tts-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive text-to-speech model with natural-language steering, voice cloning and cross-lingual synthesis", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/ogg", "audio/alaw", "audio/basic", "audio/flac", "audio/pcm" ] }, "name": "Realtime TTS-2", "open_weights": false, "release_date": "2026-05-05", "schema": 2, "type": "speech", "v2_only": true }, "jaredpalmer/kev-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Jared Palmer's open-weight System One model, fine-tuned from Qwen3.5-4B-Base, that returns calibrated probabilities for typed choice, score and yes/no questions about shared text state", "family": "qwen", "license": "Apache-2.0", "limit": { "context": 65536, "output": 0 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/jaredpalmer/kev-4b" } ], "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Kev 4B", "open_weights": true, "release_date": "2026-09-24", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/jaredpalmer/kev-4b" } ] }, "k2-fsa/omnivoice": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Massively multilingual zero-shot text-to-speech model with preset, described or cloned voices", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Speech generation checkpoint", "notes": "F32 tensor-header weight count; excludes the eight I64 codebook offsets and the separate audio tokenizer.", "sources": [ "https://huggingface.co/k2-fsa/OmniVoice/blob/c5fdb5ccb189668d56333f77ba2629f4cd7535f4/model.safetensors" ], "total_billion": 0.61257728 }, { "name": "Audio tokenizer", "notes": "Stored F32 weight count verified from the tokenizer's actual tensor header.", "sources": [ "https://huggingface.co/k2-fsa/OmniVoice/blob/c5fdb5ccb189668d56333f77ba2629f4cd7535f4/audio_tokenizer/model.safetensors" ], "total_billion": 0.201400553 } ], "notes": "The speech generation checkpoint and audio tokenizer are separate observations; no complete pipeline total is inferred." } } }, "modalities": { "input": [ "text" ], "output": [ "audio" ] }, "name": "OmniVoice", "open_weights": true, "release_date": "2026-03-30", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/k2-fsa/OmniVoice" } ] }, "krea/krea-2-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Krea 2 image model for photorealism and raw textures", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/png" ] }, "name": "Krea 2 Large", "open_weights": false, "release_date": "2026-05-12", "schema": 2, "type": "image", "v2_only": true }, "krea/krea-2-medium": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Balanced Krea 2 image model for general and expressive styles", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/png" ] }, "name": "Krea 2 Medium", "open_weights": false, "release_date": "2026-05-12", "schema": 2, "type": "image", "v2_only": true }, "krea/krea-2-medium-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "8-step distilled Krea 2 Medium image model for fast iteration", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Diffusion Transformer", "sources": [ "https://huggingface.co/krea/Krea-2-Turbo" ], "total_billion": 12 } ], "notes": "Only the explicitly disclosed diffusion transformer count is recorded; complete pipeline total is unknown." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/png" ] }, "name": "Krea 2 Turbo", "open_weights": true, "release_date": "2026-06-03", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/krea/Krea-2-Turbo" } ] }, "krea/krea-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast image editing model that keeps products and characters consistent", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/png" ] }, "name": "Krea 3", "open_weights": false, "release_date": "2026-10-15", "schema": 2, "type": "image", "v2_only": true }, "kuaishou/kling-image-o1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Unified Kling image model for generation and editing from text and up to 10 reference images", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://kling.ai/document-api/api/image/o1/image-generation" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image" ] }, "name": "Kling IMAGE O1", "open_weights": false, "release_date": "2025-12-15", "schema": 2, "type": "image", "v2_only": true }, "kuaishou/kling-image-v2.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling image model for text-to-image, image-to-image with subject or face reference, and multi-image composition", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://kling.ai/document-api/api/image/2-1/image-generation" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image" ] }, "name": "Kling IMAGE 2.1", "open_weights": false, "release_date": "2025-07-30", "schema": 2, "type": "image", "v2_only": true }, "kuaishou/kling-image-v3.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling 3.0 image model for text-to-image and image-to-image with element references at up to 2K", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://kling.ai/document-api/api/image/3-0-omni/image-generation" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image" ] }, "name": "Kling IMAGE 3.0", "open_weights": false, "release_date": "2026-02-05", "schema": 2, "type": "image", "v2_only": true }, "kuaishou/kling-image-v3.0-omni": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling 3.0 multimodal image model for generation, editing and image series from text and up to 10 references, with 4K output", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://kling.ai/document-api/api/image/o1/image-generation" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image" ] }, "name": "Kling IMAGE 3.0 Omni", "open_weights": false, "release_date": "2026-02-05", "schema": 2, "type": "image", "v2_only": true }, "kuaishou/kling-v2.5-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost Kling video model for text-to-video and image-to-video with first and last frame control", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 2.5 Turbo", "open_weights": false, "release_date": "2025-09-23", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v2.6": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling video model with simultaneous audio-visual generation, first and last frame control and motion control", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/quicktime" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 2.6", "open_weights": false, "release_date": "2025-12-03", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v3.0-omni": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling 3.0 multimodal video model for reference-based generation and editing with images, videos and elements, multi-shot storyboards and native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/quicktime" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 3.0 Omni", "open_weights": false, "release_date": "2026-02-05", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v3.0-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-quality Kling 3.0 video model with first and last frame control and native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 3.0 Pro", "open_weights": false, "release_date": "2026-02-05", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v3.0-std": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling 3.0 video model with first and last frame control and native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 3.0 Standard", "open_weights": false, "release_date": "2026-02-05", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v3.0-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lower-cost Kling 3.0 tier for text-to-video and first-frame image-to-video with native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 3.0 Turbo", "open_weights": false, "release_date": "2026-06-17", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v4.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Kling video model with native 30-second generation, multi-keyframe control, omni references, video editing and native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/quicktime" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 4.0", "open_weights": false, "release_date": "2026-10-15", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-v4.0-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost Kling 4.0 tier for 720P video up to 20 seconds with omni references, video editing and native audio", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/quicktime" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO 4.0 Flash", "open_weights": false, "release_date": "2026-11-04", "schema": 2, "type": "video", "v2_only": true }, "kuaishou/kling-video-o1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Unified multimodal Kling model for video generation and editing", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "video/mp4" ] }, "name": "Kling VIDEO O1", "open_weights": false, "release_date": "2025-12-01", "schema": 2, "type": "video", "v2_only": true }, "leonardo/lucid-origin": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Prompt-adherent Full HD text-to-image model with text rendering and a wide range of styles", "family": "lucid", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image" ] }, "name": "Lucid Origin", "open_weights": false, "release_date": "2025-08-05", "schema": 2, "type": "image", "v2_only": true }, "leonardo/phoenix-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Leonardo.Ai's foundational text-to-image model with strong prompt adherence and coherent text", "family": "phoenix", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image" ] }, "name": "Phoenix 1.0", "open_weights": false, "release_date": "2024-12-18", "schema": 2, "type": "image", "v2_only": true }, "levanto/levanto-sage-2": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Decision model that answers typed yes/no, choice, scale, sort and tag questions with calibrated probabilities, with streaming decisions and extraction", "limit": { "context": 32768, "output": 0 }, "links": [ { "label": "Documentation", "type": "docs", "url": "https://docs.levanto.ai/" } ], "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "Sage 2", "open_weights": false, "release_date": "2026-11-02", "schema": 2, "type": "decision", "v2_only": true }, "levanto/levanto-sage-v1.4": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Decision model that answers typed yes/no, choice, scale, sort and tag questions with calibrated probabilities", "limit": { "context": 32768, "output": 0 }, "links": [ { "label": "Documentation", "type": "docs", "url": "https://docs.levanto.ai/" } ], "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "Sage 1.4", "open_weights": false, "release_date": "2026-10-04", "schema": 2, "type": "decision" }, "lightonai/lightonocr-2-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "End-to-end 1B-parameter vision-language OCR model from the French company LightOn that converts documents (PDFs, scans, images) into clean, naturally ordered text without a brittle OCR pipeline", "license": "apache-2.0", "limit": { "context": 16384 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2601.14251" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "text" ] }, "name": "LightOnOCR 2 1B", "open_weights": true, "release_date": "2026-01-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/lightonai/LightOnOCR-2-1B" } ] }, "lightonai/lightonocr-3-0.8b": { "benchmarks": [ { "metric": "score", "name": "olmOCR-bench", "score": 85.5, "source": "https://huggingface.co/lightonai/LightOnOCR-3-0.8B" }, { "metric": "mean", "name": "ParseBench", "score": 74.6, "source": "https://huggingface.co/lightonai/LightOnOCR-3-0.8B" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "LightOnOCR-3-0.8B is the lightest model of LightOn's LightOnOCR-3 family, with the same layout and visual understanding features as its larger siblings.", "last_updated": "2026-10-14", "license": "apache-2.0", "limit": { "context": 262144, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal OCR checkpoint.", "sources": [ "https://huggingface.co/api/models/lightonai/LightOnOCR-3-0.8B" ], "total_billion": 0.85298592 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "LightOnOCR-3 0.8B", "open_weights": true, "release_date": "2026-10-08", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/lightonai/LightOnOCR-3-0.8B" } ] }, "liquid/d1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "System One decision model for typed classification, routing, and scoring over shared state", "family": "liquid", "limit": { "context": 65536, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "decision" ] }, "name": "Liquid d1", "open_weights": false, "release_date": "2026-09-29", "schema": 2, "type": "decision" }, "liquid/lfm2-24b-a2b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient open Liquid MoE instruct model for on-device chat and tool calling", "family": "liquid", "license": "LFM Open License v1.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 2.3, "notes": "Uses the architecture table's 2.3B active count, rather than the rounded 2B designation.", "sources": [ "https://huggingface.co/LiquidAI/LFM2-24B-A2B" ], "total_billion": 24 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "LFM2 24B A2B", "open_weights": true, "release_date": "2026-02-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/LiquidAI/LFM2-24B-A2B" } ] }, "liquid/lfm2.5-embedding-350m": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual text embedding model for retrieval and semantic search", "family": "liquid", "license": "LFM Open License v1.0", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete bi-encoder checkpoint, consistent with the card's approximate 354M count.", "sources": [ "https://huggingface.co/api/models/LiquidAI/LFM2.5-Embedding-350M" ], "total_billion": 0.354483968 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "LFM2.5-Embedding-350M", "open_weights": true, "release_date": "2026-06-18", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/LiquidAI/LFM2.5-Embedding-350M" } ] }, "lykon/dreamshaper-8-lcm": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Latent-consistency distilled DreamShaper 8 Stable Diffusion fine-tune for fast photorealistic images", "family": "stable-diffusion", "license": "CreativeML Open RAIL-M", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "U-Net", "notes": "Exact stored F32 weight count verified from the component's actual tensor header, not from checkpoint file size.", "sources": [ "https://huggingface.co/Lykon/dreamshaper-8-lcm/blob/4645d8bc6a8e6b106d21606d63e8460cdad4f1a6/unet/diffusion_pytorch_model.safetensors" ], "total_billion": 0.859520964 } ], "notes": "U-Net statistic only; text encoder, VAE and alternate serialization formats are not added into a model total." } } }, "modalities": { "input": [ "text" ], "output": [ "image" ] }, "name": "DreamShaper 8 LCM", "open_weights": true, "release_date": "2023-12-06", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Lykon/dreamshaper-8-lcm" } ] }, "mancer/weaver": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "General-purpose chat model for instruction following, writing, and analysis", "family": "alpha", "knowledge": "2023-06-30", "limit": { "context": 8000, "output": 6000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Weaver (alpha)", "open_weights": false, "release_date": "2023-08-02", "schema": 2 }, "meituan/longcat-2.0": { "benchmarks": [ { "date": "2026-06-30", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 59.5, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "date": "2026-06-30", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 77.3, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "date": "2026-06-30", "metric": "success rate", "name": "Terminal-Bench", "score": 70.8, "source": "https://github.com/meituan-longcat/longcat-2.0", "version": "2.1" }, { "date": "2026-06-30", "metric": "accuracy", "name": "GPQA Diamond", "score": 88.9, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "date": "2026-06-30", "metric": "accuracy", "name": "BrowseComp", "score": 79.9, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "date": "2026-06-30", "metric": "accuracy", "name": "IFEval", "score": 90.0, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "date": "2026-06-30", "metric": "success rate", "name": "FORTE", "score": 73.2, "source": "https://github.com/meituan-longcat/longcat-2.0" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-astra-low-vs-longcat-2-0", "variant": "pass@1; averaged over 3 repeats per task; maximum 500 agent steps; Artificial Analysis run", "version": "4.0.0" }, { "dataset": "2,158 text-only questions", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-astra-low-vs-longcat-2-0", "variant": "pass@1; judged by GPT-5.6 Luna (medium)" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Meituan LongCat-2.0, a reasoning model with tool calling and a 1M-token context window", "family": "longcat", "limit": { "context": 1000000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 48, "notes": "Official card reports 1.6T total and approximately 48B activated per token. Retains the author's architecture-count convention rather than inferring a larger total from serialized auxiliary modules.", "sources": [ "https://huggingface.co/meituan-longcat/LongCat-2.0" ], "total_billion": 1600 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "LongCat-2.0", "open_weights": true, "release_date": "2026-06-30", "schema": 2 }, "meituan/longcat-2.5": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Meituan's multimodal reasoning model for long-horizon coding and agent tasks, with visual understanding and a 1M-token context window", "family": "longcat", "limit": { "context": 1000000, "output": 262144 }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "LongCat-2.5", "open_weights": false, "release_date": "2026-11-02", "schema": 2 }, "meituan/longcat-2.5-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Meituan's multimodal reasoning model for coding and agent tasks, with image understanding and a 1M-token context window", "family": "longcat", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "LongCat-2.5-Preview", "open_weights": false, "release_date": "2026-09-25", "schema": 2 }, "meragpt/decider-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "meraGPT decision model that answers typed yes/no, choice and score questions about a JSON state with calibrated probabilities", "limit": { "context": 4096, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Decider 1", "open_weights": false, "release_date": "2026-09-22", "schema": 2, "type": "decision", "v2_only": true }, "meragpt/restyler-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "meraGPT model that rewrites machine-written prose so it reads as if a person wrote it, keeping its structure", "limit": { "context": 4096, "output": 2048 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Restyler 1", "open_weights": false, "release_date": "2026-09-02", "schema": 2 }, "meta/llama-3.1-70b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "llama", "knowledge": "2023-12", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint, not the rounded size designation.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.1-70B-Instruct" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-3.1-70B-Instruct", "open_weights": true, "release_date": "2024-07-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct" } ] }, "meta/llama-3.1-8b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact open Llama model for lightweight chat, drafting, and self-hosting", "family": "llama", "knowledge": "2023-12", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.1-8B-Instruct" ], "total_billion": 8.030261248 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-3.1-8B-Instruct", "open_weights": true, "release_date": "2024-07-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct" } ] }, "meta/llama-3.2-11b-vision-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal Llama model for image understanding, captioning, and visual QA", "family": "llama", "knowledge": "2023-12", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not the rounded size designation.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-11B-Vision-Instruct" ], "total_billion": 10.670220835 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Llama-3.2-11B-Vision-Instruct", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct" } ] }, "meta/llama-3.2-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Compact open Llama base model for lightweight and on-device use", "family": "llama", "knowledge": "2023-12", "license": "Llama 3.2 Community License", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete base checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-1B" ], "total_billion": 1.2358144 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-3.2-1B", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-1B" } ] }, "meta/llama-3.2-1b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact instruction-tuned Llama model for multilingual dialogue and on-device assistants", "family": "llama", "knowledge": "2023-12", "license": "Llama 3.2 Community License", "limit": { "context": 131072, "output": 32000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete instruction-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-1B-Instruct" ], "total_billion": 1.2358144 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.2 1B Instruct", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct" } ] }, "meta/llama-3.2-3b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Small open Llama base model for lightweight text generation and self-hosting", "family": "llama", "knowledge": "2023-12", "license": "Llama 3.2 Community License", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete base checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-3B" ], "total_billion": 3.212749824 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-3.2-3B", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-3B" } ] }, "meta/llama-3.2-3b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Small instruction-tuned Llama model for multilingual dialogue and agentic retrieval", "family": "llama", "knowledge": "2023-12", "license": "Llama 3.2 Community License", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for this complete instruction-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-3B-Instruct" ], "total_billion": 3.212749824 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.2 3B Instruct", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct" } ] }, "meta/llama-3.2-90b-vision-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Large open multimodal Llama model for image reasoning, document QA, and captioning", "family": "llama", "knowledge": "2023-12", "license": "Llama 3.2 Community License", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not the rounded size designation.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.2-90B-Vision-Instruct" ], "total_billion": 88.593355323 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Llama-3.2-90B-Vision-Instruct", "open_weights": true, "release_date": "2024-09-25", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.2-90B-Vision-Instruct" } ] }, "meta/llama-3.3-70b-instruct": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 10.7, "source": "https://openrouter.ai/meta-llama/llama-3.3-70b-instruct/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 26, "source": "https://openrouter.ai/meta-llama/llama-3.3-70b-instruct/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 3, "source": "https://openrouter.ai/meta-llama/llama-3.3-70b-instruct/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Popular open Llama workhorse for multilingual chat, coding, and self-hosting", "family": "llama", "knowledge": "2023-12", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-3.3-70B-Instruct" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-3.3-70B-Instruct", "open_weights": true, "release_date": "2024-12-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" } ] }, "meta/llama-4-maverick-17b-instruct": { "benchmarks": [ { "date": "2025-04-06", "metric": "percent correct", "name": "Aider Polyglot", "score": 15.6, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 5.24, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 5.68, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal Llama for strong reasoning with efficient everyday serving", "family": "llama", "knowledge": "2024-08", "limit": { "context": 1000000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 17, "notes": "Official Model Information table's rounded total, not the 17B activated-parameter designation.", "sources": [ "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" ], "total_billion": 400 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "Llama 4 Maverick 17B Instruct", "open_weights": true, "release_date": "2025-04-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" } ] }, "meta/llama-4-scout-17b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Llama with long-context vision for efficient multimodal agents", "family": "llama", "knowledge": "2024-08", "limit": { "context": 10000000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 17, "sources": [ "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" ], "total_billion": 109 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "Llama 4 Scout 17B Instruct", "open_weights": true, "release_date": "2025-04-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" } ] }, "meta/llama-guard-3-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Llama 3.1-based safety classifier for moderating prompts and model responses", "family": "llama", "knowledge": "2023-12", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-Guard-3-8B" ], "total_billion": 8.030261248 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama-Guard-3-8B", "open_weights": true, "release_date": "2024-07-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-Guard-3-8B" } ] }, "meta/llama-guard-4-12b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multimodal Llama 4-based safety classifier for moderating prompts and responses", "family": "llama", "license": "Llama 4 Community License", "limit": { "context": 163840, "output": 163840 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/meta-llama/Llama-Guard-4-12B" ], "total_billion": 12 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Llama Guard 4 12B", "open_weights": true, "release_date": "2025-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-Guard-4-12B" } ] }, "meta/llama-prompt-guard-2-22m": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "22M-parameter (DeBERTa-xsmall) English classifier that detects prompt injection and jailbreak attempts", "family": "llama", "license": "Llama 4 Community License", "limit": { "context": 512, "output": 512 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the single complete unquantized checkpoint. Uses the full stored-weight count, not the model card's nominal 22M size designation.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-Prompt-Guard-2-22M" ], "total_billion": 0.070830722 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama Prompt Guard 2 22M", "open_weights": true, "release_date": "2025-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M" } ] }, "meta/llama-prompt-guard-2-86m": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "86M-parameter (mDeBERTa-base) multilingual classifier that detects prompt injection and jailbreak attempts", "family": "llama", "license": "Llama 4 Community License", "limit": { "context": 512, "output": 512 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the single complete unquantized checkpoint. Uses the full stored-weight count, not the nominal 86M size designation.", "sources": [ "https://huggingface.co/api/models/meta-llama/Llama-Prompt-Guard-2-86M" ], "total_billion": 0.278810882 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama Prompt Guard 2 86M", "open_weights": true, "release_date": "2025-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M" } ] }, "meta/muse-ember-1.0": { "benchmarks": [ { "dataset": "Zapier private set (657 tasks)", "effort": "Max", "metric": "strict pass@1", "name": "AutomationBench-AA", "score": 37.8, "source": "https://research.meta.ai/blog/muse-ember" }, { "dataset": "Linux split (100 of 105 tasks)", "effort": "Max", "harness": "Meta common internal agent loop", "metric": "mean task score", "name": "Agents' Last Exam", "score": 61.6, "source": "https://research.meta.ai/blog/muse-ember", "variant": "5 tasks excluded for storage limits or unscored evaluation scripts; native partial-credit scoring", "version": "1.0" }, { "effort": "Max", "metric": "index score", "name": "Personal Superintelligence Index", "score": 46.8, "source": "https://research.meta.ai/blog/muse-ember", "variant": "Meta internal composite of eight personal and knowledge-work evaluations" }, { "dataset": "Full (66 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "accuracy", "name": "Terminal-Bench", "score": 40.9, "source": "https://research.meta.ai/blog/muse-ember", "variant": "Artificial Analysis run; launch scorecard uses each model's best published harness", "version": "4.0.0" }, { "effort": "Max", "harness": "Muse Code", "metric": "index score", "name": "Artificial Analysis Coding Agent Index", "score": 46, "source": "https://research.meta.ai/blog/muse-ember", "version": "2.0" }, { "dataset": "Artificial Analysis private set (132 tasks)", "effort": "Max", "metric": "accuracy", "name": "FrontierCode 2.0-AA", "score": 27.2, "source": "https://research.meta.ai/blog/muse-ember", "version": "2.0" }, { "effort": "Max", "metric": "all-pass rate", "name": "GDP.pdf-AA", "score": 22.8, "source": "https://research.meta.ai/blog/muse-ember", "variant": "extracted PDF text and page images; without tools; every rubric criterion must pass" }, { "dataset": "108 real-world workflows", "effort": "Max", "harness": "Meta common internal framework", "metric": "mean partial score", "name": "OSWorld", "score": 69.5, "source": "https://research.meta.ai/blog/muse-ember", "version": "osworld-v2.1" }, { "dataset": "108 real-world workflows", "effort": "Max", "harness": "Meta common internal framework", "metric": "binary task completion rate", "name": "OSWorld", "score": 33.8, "source": "https://research.meta.ai/blog/muse-ember", "version": "osworld-v2.1" }, { "dataset": "Full (3000 questions)", "effort": "Max", "harness": "Meta common internal framework", "metric": "pass@1", "name": "PerceptionBench", "score": 80.6, "source": "https://research.meta.ai/blog/muse-ember", "variant": "with tools: bash terminal, file viewer and editor; official gpt-oss-120b judge and prompt" }, { "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 36, "source": "https://research.meta.ai/blog/muse-ember", "version": "5.0" }, { "cost": 0.1359356168986363, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 36.0212987864912, "source": "https://artificialanalysis.ai/models/muse-ember", "version": "5.0.1" }, { "cost": 0.26582015270981807, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 40.9090909090909, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 0.4328794980292568, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 20, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 0.017808669803981735, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Max", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 68.07901586086508, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "cost": 0.1162277983030469, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34.1149608609788, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "version": "5.0.1" }, { "cost": 0.22196078406884848, "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 35.3535353535354, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 0.39906039476754307, "cost_unit": "USD/task", "dataset": "70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 17.6190476190476, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 0.016065550956663625, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Xhigh", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 66.70632837564682, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "zero for any guardrail violation, otherwise share of objectives completed; one run per task; API toolset; 50-turn cap", "version": "1.0.6" }, { "cost": 0.15305, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.DeTlTtu4.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 60.475, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 0.059873, "cost_source": "https://www.vals.ai/_astro/benchmark_view_finance_index.SoS3fFjr.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Finance Index", "score": 60.914, "source": "https://www.vals.ai/benchmarks/finance_index", "version": "1" }, { "cost": 0.068939, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_index.DoifPxRC.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Legal Index", "score": 75.686, "source": "https://www.vals.ai/benchmarks/legal_index", "version": "1" }, { "cost": 0.091434, "cost_source": "https://www.vals.ai/_astro/benchmark_view_emb_v1_1.BepKO0Tl.json", "cost_unit": "USD/test", "effort": "Max", "metric": "modeling score", "name": "EMB v1.1", "score": 72.169, "source": "https://www.vals.ai/benchmarks/emb-v1-1", "variant": "Template cell matching and Scratch rubric scoring; GPT-6 Luna Scratch judge; equal category weights; 3.5-hour task limit", "version": "1.1" }, { "cost": 0.056396, "cost_source": "https://www.vals.ai/_astro/benchmark_view_finance_agent_v2_1.6b1PiRZr.json", "cost_unit": "USD/test", "effort": "Max", "metric": "partial credit", "name": "Finance Agent v2.1", "score": 63.834, "source": "https://www.vals.ai/benchmarks/fabv2-1", "variant": "dealbreaker-gated, severity-weighted checks; three-judge jury; two-hour task limit", "version": "2.1" }, { "cost": 0.056581, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_research.BeXLzS5J.json", "cost_unit": "USD/test", "dataset": "Private Test split (208 questions)", "effort": "Max", "metric": "all-pass accuracy", "name": "Legal Research Bench", "score": 49.038, "source": "https://www.vals.ai/benchmarks/legal_research", "variant": "GPT 5.4 rubric judge; three-hour task limit", "version": "1" }, { "cost": 0.031788, "cost_source": "https://www.vals.ai/_astro/benchmark_view_tax_agent_bench.CoAIMgK7.json", "cost_unit": "USD/test", "dataset": "Private Test split (193 questions)", "effort": "Max", "metric": "citation-adjusted all-pass accuracy", "name": "Tax Agent Bench", "score": 38.233, "source": "https://www.vals.ai/benchmarks/tax_agent_bench", "variant": "all-pass times (0.7 + 0.3 times citation quality); GPT 5.4 rubric judge; three-hour task limit", "version": "1" }, { "cost": 0.054457, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vibe_code_bench.DtQxXDu0.json", "cost_unit": "USD/test", "effort": "Max", "harness": "OpenHands", "metric": "test pass rate", "name": "Vibe Code Bench v1.1", "score": 83.703, "source": "https://www.vals.ai/benchmarks/vibe-code", "version": "1.1" }, { "cost": 0.118448, "cost_source": "https://www.vals.ai/_astro/benchmark_view_legal_agent_benchmark.BNvE9afF.json", "cost_unit": "USD/test", "dataset": "120 tasks", "effort": "Max", "metric": "all-pass rate", "name": "HLAB-Vals-1.1", "score": 25, "source": "https://www.vals.ai/benchmarks/hlab", "variant": "Harvey v1.1 holdout fixes; Vals harness updates; offline; GPT 5.5 and Claude Sonnet 4.6 judges", "version": "1.1" }, { "cost": 0.218381, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.D_ldiP5A.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 39.394, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); eight-hour task limit; no step or cost limit", "version": "4.0.0" }, { "cost": 0.13473978901731876, "cost_unit": "USD/task", "effort": "Max", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1520.92, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 0.09612423523006064, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Max", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 27.1696212121212, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.001460120233333333, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Max", "metric": "Net Score", "name": "AA-Omniscience", "score": 7.366666666666666, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 0.017274625000000002, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Max", "metric": "all-pass rate", "name": "GDP.pdf", "score": 22.8, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.024368178900000006, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Max", "metric": "accuracy", "name": "HLE-Diamond", "score": 25.1, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 0.0538207262096, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Max", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 39, "source": "https://artificialanalysis.ai/models/muse-ember", "variant": "five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" }, { "cost": 0.10155680184228572, "cost_unit": "USD/task", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo", "name": "AA-Briefcase", "score": 1526.95, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "combined Elo; offline; 500-turn cap", "version": "1.1" }, { "cost": 0.07630167588969705, "cost_unit": "USD/task", "dataset": "132 tasks", "effort": "Xhigh", "harness": "Stirrup", "metric": "weighted rubric score", "name": "FrontierCode", "score": 21.974898989899, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "three attempts per task; blocking-check and anti-cheat gating; GPT-6 Luna (medium) judge; 500-turn cap", "version": "2.0" }, { "cost": 0.0010508426999999998, "cost_unit": "USD/task", "dataset": "3,000 questions", "effort": "Xhigh", "metric": "Net Score", "name": "AA-Omniscience", "score": 7.733333333333333, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "GPT-5.6 Luna (medium) judge", "version": "1.1" }, { "cost": 0.015790040999999998, "cost_unit": "USD/task", "dataset": "100 tasks; 500 attempts", "effort": "Xhigh", "metric": "all-pass rate", "name": "GDP.pdf", "score": 22.4, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "extracted text and adapted page images; no tools; five attempts per task; GPT-5.6 Luna (medium) criterion judge" }, { "cost": 0.016926525600000002, "cost_unit": "USD/task", "dataset": "1,000 questions, including 272 image questions", "effort": "Xhigh", "metric": "accuracy", "name": "HLE-Diamond", "score": 23.6, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "cost": 0.040508304584, "cost_unit": "USD/task", "dataset": "Fixed private hard subset (40 of 70 problems)", "effort": "Xhigh", "harness": "Stirrup", "metric": "accuracy", "name": "CritPt v2 (Hard Subset)", "score": 38.5, "source": "https://artificialanalysis.ai/models/muse-ember-xhigh", "variant": "five repeats, mean pass@1; code tools; offline; 50-turn cap; November 2026 regrading", "version": "2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Ember is a multimodal reasoning model from Meta for high-volume, repeated agentic work, including coding subtasks, tool calls, routing, document summarization, and extraction. It accepts text, images, audio, video, and PDFs and outputs text.", "family": "muse", "limit": { "context": 1048576, "output": 524288 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://research.meta.ai/blog/muse-ember" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "application/pdf", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Muse Ember", "open_weights": false, "release_date": "2026-11-09", "schema": 2 }, "meta/muse-glimmer-30b": { "benchmarks": [ { "date": "2026-08-10", "metric": "success rate", "name": "MCP Atlas", "score": 75.5, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "variant": "public" }, { "date": "2026-08-10", "name": "DeepSearch QA", "score": 74.6, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "date": "2026-08-10", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 51.2, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "date": "2026-08-10", "metric": "resolve rate", "name": "SWE-Bench Verified", "score": 76.0, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "date": "2026-08-10", "metric": "success rate", "name": "Terminal-Bench", "score": 51.7, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "variant": "with terminus2", "version": "2.1" }, { "date": "2026-08-10", "metric": "success rate", "name": "OSWorld-Verified", "score": 65.9, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "date": "2026-08-10", "metric": "accuracy", "name": "AIME 2026", "score": 94.7, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "date": "2026-08-10", "metric": "accuracy", "name": "GPQA Diamond", "score": 83.5, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "variant": "AA" }, { "date": "2026-08-10", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 78.8, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "cost": 0.2824037170303031, "cost_source": "https://artificialanalysis.ai/models/muse-glimmer", "cost_unit": "USD/task", "dataset": "Terminal-Bench 4.0 (66 tasks)", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1.0, "source": "https://artificialanalysis.ai/models/comparisons/muse-glimmer-vs-hy3", "variant": "Artificial Analysis independent run; pass@1 averaged over 3 repeats", "version": "4.0.0" }, { "dataset": "100 tasks", "effort": "xHigh", "harness": "Inspect AI; Gemini 3.5 Flash judge", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 17.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "xHigh reasoning; mean of 10 runs per task" }, { "dataset": "text-only; 2,158 questions", "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 22.0, "source": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "variant": "no tools; Artificial Analysis result" }, { "dataset": "220 tasks; 44 occupations; 9 industries", "effort": "High", "harness": "Artificial Analysis Stirrup; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 793, "source": "https://artificialanalysis.ai/models/comparisons/muse-glimmer-vs-hy3", "version": "2.1" }, { "cost": 0.16096168039127723, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 6.34128724913462, "source": "https://artificialanalysis.ai/models/muse-glimmer", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Muse Glimmer is a 30-billion-parameter open-weight multimodal model from Meta Superintelligence Labs, distilled from Muse Spark for always-on local agents, tool use, coding, and image understanding.", "family": "muse", "knowledge": "2026-01-04", "license": "Apache 2.0", "limit": { "context": 131072, "output": 131072 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://research.meta.ai/blog/introducing-muse-glimmer-open-agentic-model" }, { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "label": "Developer docs", "type": "docs", "url": "https://developer.meta.com/ai/models/muse-glimmer/" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Approximately 29.6B including the vision encoder, as explicitly specified in the model card.", "sources": [ "https://huggingface.co/meta-models/Muse-Glimmer-30B" ], "total_billion": 29.6 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Muse Glimmer 30B", "open_weights": true, "release_date": "2026-08-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B" } ] }, "meta/muse-image-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Agentic image generation and editing model that plans, searches and refines before rendering", "family": "muse", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png", "image/jpeg" ] }, "name": "Muse Image", "open_weights": false, "release_date": "2026-07-07", "schema": 2, "type": "image", "v2_only": true }, "meta/muse-spark-1.1": { "benchmarks": [ { "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 61.5, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "metric": "success rate", "name": "Terminal-Bench", "score": 80.0, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/", "version": "2.1" }, { "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent internal bash-only fork; 64 GB RAM and storage VM", "metric": "pass@1", "name": "DeepSWE", "score": 53.3, "source": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report", "variant": "pass@1 averaged over five attempts; internet disabled", "version": "1.1" }, { "date": "2026-07-09", "metric": "success rate", "name": "MCP Atlas", "score": 88.1, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "date": "2026-07-09", "metric": "success rate", "name": "JobBench", "score": 54.7, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Toolathlon-Verified", "score": 75.6, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "dataset": "full 2,500-question HLE", "effort": "Xhigh", "harness": "Meta Model API; GPT o3-mini judge", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 62.1, "source": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report", "variant": "with tools; bash and browser; blocklist huggingface.co, kaggle.com, paperswithcode.com, lmsys.org" }, { "date": "2026-07-09", "metric": "success rate", "name": "OSWorld-Verified", "score": 80.8, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "date": "2026-07-09", "metric": "accuracy", "name": "Finance Agent", "score": 57.2, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/", "version": "v2" }, { "date": "2026-07-09", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 88.4, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "date": "2026-07-09", "metric": "accuracy", "name": "BabyVision", "score": 76.3, "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/" }, { "dataset": "220 tasks across 44 occupations", "effort": "Xhigh", "harness": "Artificial Analysis Stirrup; E2B sandbox with shell/code execution and web tools", "metric": "Elo rating", "name": "GDPval-AA", "score": 1212, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-1-vs-gpt-5-4", "version": "2.1" }, { "dataset": "100 tasks", "harness": "Inspect AI leaderboard configuration; 10 runs per task; Gemini 3.5 Flash judge", "name": "Chartography", "score": 24.4, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "xHigh reasoning" }, { "dataset": "full 2,500-question HLE", "effort": "Xhigh", "harness": "Meta Model API; GPT o3-mini judge", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 52.2, "source": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report", "variant": "no tools" }, { "cost": 2.361142644469026, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.31858407079646, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Spark is a natively multimodal reasoning model with support for tool-use, visual chain of thought, and multi-agent orchestration.", "family": "muse", "last_updated": "2026-07-09", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf", "video/mp4" ], "output": [ "text" ] }, "name": "Muse Spark 1.1", "open_weights": false, "release_date": "2026-04-08", "schema": 2 }, "meta/muse-spark-1.2": { "benchmarks": [ { "date": "2026-09-02", "harness": "Stirrup", "metric": "Elo", "name": "GDPval-AA", "score": 1615, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2" }, { "date": "2026-09-02", "harness": "OpenCode", "metric": "mean rubric score", "name": "JobBench", "score": 61.6, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort" }, { "dataset": "06.24", "date": "2026-09-02", "harness": "Meta internal evaluation framework", "metric": "mean partial score", "name": "OSWorld", "score": 47.6, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2.0" }, { "dataset": "06.24", "date": "2026-09-02", "harness": "Meta internal evaluation framework", "metric": "binary completion rate", "name": "OSWorld", "score": 17.9, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2.0" }, { "date": "2026-09-02", "harness": "Meta browser harness", "metric": "F1", "name": "DeepSearchQA", "score": 85.9, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort" }, { "dataset": "Meta internal", "date": "2026-09-02", "metric": "score", "name": "Agentic IF Index", "score": 46.2, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort" }, { "dataset": "public v3 task set", "date": "2026-09-02", "metric": "pass@1", "name": "AutomationBench", "score": 38.2, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort" }, { "dataset": "8-needle, 256K-512K", "date": "2026-09-02", "metric": "mean sequence-match ratio", "name": "MRCR", "score": 66.3, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2" }, { "dataset": "8-needle, 512K-1M", "date": "2026-09-02", "metric": "mean sequence-match ratio", "name": "MRCR", "score": 55.5, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2" }, { "date": "2026-09-02", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 55.0, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "1.1" }, { "date": "2026-09-02", "harness": "mini-swe-agent", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 46.2, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort" }, { "date": "2026-09-02", "harness": "Muse Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 82.9, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "xhigh effort", "version": "2.1" }, { "date": "2026-08-05", "harness": "Muse Code", "metric": "pass@1", "name": "DeepSWE", "score": 59.3, "source": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2", "variant": "xhigh effort", "version": "1.1" }, { "date": "2026-08-05", "harness": "Scale AI", "metric": "pass rate", "name": "MCP Atlas", "score": 90.3, "source": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2", "variant": "xhigh effort" }, { "dataset": "220 tasks", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1474, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-2-vs-gpt-5-4", "version": "2.1" }, { "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 7, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-muse-spark-1-2", "variant": "pass@1; three repeats per task", "version": "4.0.0" }, { "dataset": "100-task full Chartography set", "effort": "xHigh", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 32.1, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "cost": 3.6955048180309733, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 54.86725663716814, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 4.135537, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 6.061, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Spark 1.2 is a coding-focused update to Muse Spark 1.1 with improvements in code generation, complex debugging, codebase understanding, and end-to-end developer workflows.", "family": "muse", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "application/pdf", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Muse Spark 1.2", "open_weights": false, "release_date": "2026-08-05", "schema": 2 }, "meta/muse-spark-1.3": { "benchmarks": [ { "harness": "Stirrup", "metric": "Elo", "name": "GDPval-AA", "score": 1754, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2" }, { "harness": "OpenCode", "metric": "mean rubric score", "name": "JobBench", "score": 64.9, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort" }, { "dataset": "08.08", "harness": "Meta internal evaluation framework", "metric": "mean partial score", "name": "OSWorld", "score": 66.9, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2.0" }, { "dataset": "08.08", "harness": "Meta internal evaluation framework", "metric": "binary completion rate", "name": "OSWorld", "score": 32.0, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2.0" }, { "harness": "Meta browser harness", "metric": "F1", "name": "DeepSearchQA", "score": 90.3, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort" }, { "dataset": "Meta internal", "metric": "score", "name": "Agentic IF Index", "score": 57.8, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort" }, { "dataset": "public v3 task set", "metric": "pass@1", "name": "AutomationBench", "score": 49.6, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort" }, { "dataset": "8-needle, 256K-512K", "metric": "mean sequence-match ratio", "name": "MRCR", "score": 98.5, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2" }, { "dataset": "8-needle, 512K-1M", "metric": "mean sequence-match ratio", "name": "MRCR", "score": 98.1, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2" }, { "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 75.4, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "1.1" }, { "harness": "mini-swe-agent", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 59.4, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort" }, { "harness": "Muse Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 88.8, "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "variant": "max effort", "version": "2.1" }, { "cost": 5.583529694146059, "cost_source": "https://artificialanalysis.ai/models/muse-spark-1-3", "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 33, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-muse-spark-1-3-xhigh", "variant": "mean of 3 repeats per task", "version": "4.0.0" }, { "dataset": "66 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 17, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-muse-spark-1-3-xhigh", "variant": "mean of 3 repeats per task", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 11, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-muse-spark-1-3-xhigh", "variant": "mean of 3 repeats per task", "version": "0.1.0" }, { "dataset": "70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 4, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-muse-spark-1-3-xhigh", "variant": "mean of 3 repeats per task", "version": "0.1.0" }, { "dataset": "657 held-out workflows", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 20.7, "source": "https://fireworks.ai/specialized-intelligence-index/benchmarks/workflow-automation/automationbench", "variant": "API mode; 50-step cap", "version": "1.0.6" }, { "dataset": "108 tasks", "effort": "Max", "metric": "mean partial score", "name": "OSWorld", "score": 69.6, "source": "https://www.aib.vote/en/benchmarks/osworld-2-1", "variant": "batch tool; up to 500 actions", "version": "v2.1" }, { "dataset": "108 tasks", "effort": "Max", "metric": "binary task completion rate", "name": "OSWorld", "score": 39.3, "source": "https://www.aib.vote/en/benchmarks/osworld-2-1", "variant": "batch tool; up to 500 actions", "version": "v2.1" }, { "dataset": "100 expert-authored tasks", "effort": "xHigh reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 27.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "dataset": "original HLE; 2,500 questions", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 49, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-gpt-5-6-sol-high" }, { "dataset": "original HLE; 2,500 questions", "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 47, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-xhigh-vs-inkling" }, { "cost": 2.3857813858107244, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 36.6172039139368, "source": "https://artificialanalysis.ai/models/muse-spark-1-3", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1681, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-vs-gpt-5-6-sol-high", "variant": "shell and web access; blind pairwise comparisons", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1627, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-1-3-xhigh-vs-inkling", "variant": "shell and web access; blind pairwise comparisons", "version": "2.1" }, { "cost": 6.654565, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 24.747, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "cost": 7.036467, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 10.606, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Spark 1.3 is a multimodal reasoning model from Meta for long-running agentic, multi-agent, and coding workflows. It improves long-horizon agent collaboration, instruction following, and coding efficiency relative to Muse Spark 1.2.", "family": "muse", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "application/pdf", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Muse Spark 1.3", "open_weights": false, "release_date": "2026-09-02", "schema": 2 }, "meta/muse-spark-open": { "benchmarks": [ { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 39, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort", "version": "4.3" }, { "date": "2026-10-15", "name": "JobBench", "score": 61.8, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "name": "MCP Atlas", "score": 89.2, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1401, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort", "version": "2" }, { "date": "2026-10-15", "name": "OSWorld-Verified", "score": 80.4, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "name": "Toolathlon-Verified", "score": 72.8, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "name": "HLE-Diamond", "score": 26.1, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, no tools" }, { "date": "2026-10-15", "name": "AA-Omniscience", "score": 25, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "name": "SciCode", "score": 58.5, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 61.0, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, with tools", "version": "unspecified" }, { "date": "2026-10-15", "name": "GPQA Diamond", "score": 89.0, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "effort": "Xhigh", "harness": "Muse Code", "metric": "pass@1", "name": "DeepSWE", "score": 60, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "version": "1.1" }, { "dataset": "V2 Full/default split, 642 tasks; reported 95.02 ± 1.43", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 95.02, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "cost": 7.264062, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0, 66 tasks (Vals AI evaluation)", "effort": "Xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 6.566, "source": "https://www.vals-ai.com/benchmarks/terminal-bench-4", "variant": "Vals AI run; reported ±0.51", "version": "4.0.0" }, { "cost": 4.656965702348483, "cost_source": "https://artificialanalysis.ai/models/muse-spark-open", "cost_unit": "USD/task", "dataset": "Terminal-Bench 4.0 (Artificial Analysis evaluation)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 8, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-open-vs-gpt-6-1-sol-high", "variant": "Artificial Analysis run; displayed rounded to integer percent", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 2, "source": "https://artificialanalysis.ai/models/comparisons/muse-spark-open-vs-gpt-6-1-sol-high", "variant": "pass@1 averaged over 3 repeats; source displays rounded integer percent", "version": "0.1.0" }, { "cost": 4.71792, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Xhigh", "metric": "performance score", "name": "Vals Index", "score": 48.567, "source": "https://www.vals.ai/models/meta_muse-spark-open", "version": "2.2" }, { "date": "2026-10-15", "harness": "preferred harness", "name": "SWE-Atlas Codebase QnA", "score": 58, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "harness": "Terminus 2", "name": "Terminal-Bench", "score": 76.4, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort", "version": "2.1" }, { "date": "2026-10-15", "name": "ZeroBench", "score": 50.0, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, with tools" }, { "date": "2026-10-15", "name": "BabyVision", "score": 80.2, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, with tools" }, { "date": "2026-10-15", "name": "SimpleVQA", "score": 73.6, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, with tools" }, { "date": "2026-10-15", "name": "WorldVQA", "score": 49.4, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort, with tools" }, { "date": "2026-10-15", "name": "IntPhys2", "score": 68.3, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort" }, { "date": "2026-10-15", "name": "Video-MME", "score": 61.5, "source": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "variant": "xhigh effort", "version": "2" }, { "cost": 2.0176680435839316, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 24.8815868992913, "source": "https://artificialanalysis.ai/models/muse-spark-open", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Muse Spark Open is an 816B total / 42B active open-weight multimodal reasoning model from Meta for coding and long-horizon agentic work, with 1M-token context.", "family": "muse", "knowledge": "2026-01-04", "license": "Muse Model License", "limit": { "context": 1048576, "output": 131072 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://research.meta.ai/blog/muse-spark-open" }, { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B" }, { "label": "Developer docs", "type": "docs", "url": "https://dev.meta.ai/docs/muse-spark-open" }, { "label": "License", "type": "license", "url": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B/blob/main/LICENSE.md" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published total for the complete unquantized BF16/F32 multimodal checkpoint; not the nominal 816B language-model designation.", "sources": [ "https://huggingface.co/api/models/meta-models/Muse-Spark-Open-816B-A42B" ], "total_billion": 822.036529664 } } }, "modalities": { "input": [ "text", "image", "video", "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "Muse Spark Open", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "weights": [ { "label": "Hugging Face (BF16)", "url": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B" }, { "label": "Hugging Face (NVFP4)", "url": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B-NVFP4" } ] }, "meta/muse-voice-transcribe-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with speaker diarization and endpointing", "family": "muse", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Muse Voice Transcribe 1.0", "open_weights": false, "release_date": "2026-09-01", "schema": 2, "type": "transcription", "v2_only": true }, "meta/muse-wonder-1.0": { "benchmarks": [ { "date": "2026-10-12", "metric": "all-pass rate", "name": "Harvey Legal Agent Benchmark", "score": 10.0, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis LAB-AA", "version": "1.1" }, { "dataset": "Vals AI private test set", "date": "2026-10-12", "metric": "accuracy", "name": "Finance Agent", "score": 56.6, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "v2" }, { "date": "2026-10-12", "metric": "length-normalized weighted rubric score", "name": "HealthBench Professional", "score": 60.5, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "dataset": "Artificial Analysis AutomationBench-AA held-out private set (657 tasks)", "metric": "strict pass@1", "name": "AutomationBench", "score": 39.1, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis AutomationBench-AA", "version": "1.0.6" }, { "date": "2026-10-12", "metric": "score", "name": "GDP.pdf", "score": 30.8, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "date": "2026-10-12", "metric": "mean reward", "name": "JobBench", "score": 70.5, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "metric": "mean partial score", "name": "OSWorld", "score": 72.6, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "osworld-v2.1" }, { "metric": "binary task completion rate", "name": "OSWorld", "score": 34.3, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; source labels this binary completion rate", "version": "osworld-v2.1" }, { "metric": "accuracy", "name": "Terminal-Bench", "score": 52.0, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis Terminal-Bench 4.0-AA", "version": "4.0.0" }, { "metric": "pass@1", "name": "DeepSWE", "score": 78.8, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "1.1" }, { "date": "2026-10-12", "metric": "pass@1", "name": "SWE-Marathon", "score": 53.1, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "date": "2026-10-12", "harness": "mini-swe-agent", "metric": "share of rollouts passing at least 95% of tests", "name": "ProgramBench", "score": 50.0, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 52, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "4.3" }, { "dataset": "SWE-bench Pro V2 Full (642 tasks); reported 96.73 ± 1.12%", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 96.73, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "Full", "version": "2.0.0" }, { "cost": 14.657694, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0 (66 tasks); reported ±2.20 percentage points", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 49.495, "source": "https://www.vals.ai/models/meta_muse-wonder", "variant": "Vals AI independent run; source labels score accuracy; avg@3 (each run pass@1)", "version": "4.0.0" }, { "dataset": "Terminal-Bench Science v0.1.0 (70 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 26, "source": "https://artificialanalysis.ai/models/comparisons/muse-wonder-vs-gpt-6-astra", "variant": "Artificial Analysis run; source labels score accuracy; pass@1 averaged over 3 repeats; score displayed as an integer percent", "version": "0.1.0" }, { "dataset": "Terminal-Bench Science v0.1.0 (70 tasks); reported ±5.05 percentage points", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 22.86, "source": "https://www.vals.ai/models/meta_muse-wonder", "variant": "Vals AI run; source labels score accuracy; single pass@1", "version": "0.1.0" }, { "dataset": "GDPval-AA v2.1 (220 tasks; 44 occupations; 9 industries)", "effort": "Max", "harness": "Stirrup agent loop (shell and web browsing)", "metric": "Elo rating", "name": "GDPval-AA", "score": 1770, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 10.647504, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "Vals Index composite (finance, coding, legal, tax)", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 64.877, "source": "https://www.vals.ai/models/meta_muse-wonder", "variant": "model profile reports ±1.07", "version": "2.2" }, { "cost": 4.560546278670271, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 40.5266079185604, "source": "https://artificialanalysis.ai/models/muse-wonder-high", "version": "5.0.1" }, { "cost": 6.9520364786062245, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 43.5748225998431, "source": "https://artificialanalysis.ai/models/muse-wonder", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 50.1, "source": "https://artificialanalysis.ai/models/muse-wonder", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 39.0, "source": "https://artificialanalysis.ai/models/muse-wonder", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Wonder is a frontier multimodal reasoning model from Meta for personal and professional agents, long-horizon coding, and computer use. It combines Active Memory, which carries relevant context across conversations, with Active Focus, which keeps it working through blockers toward a goal.", "family": "muse", "limit": { "context": 1048576, "output": 131072 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://research.meta.ai/blog/muse-wonder" }, { "label": "Model card", "type": "model_card", "url": "https://research.meta.ai/static/muse-wonder-model-card" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "application/pdf", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Muse Wonder", "open_weights": false, "release_date": "2026-10-12", "schema": 2 }, "meta/muse-wonder-full": { "benchmarks": [ { "date": "2026-10-12", "metric": "all-pass rate", "name": "Harvey Legal Agent Benchmark", "score": 13.1, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis LAB-AA", "version": "1.1" }, { "dataset": "Vals AI private test set", "date": "2026-10-12", "metric": "accuracy", "name": "Finance Agent", "score": 59.1, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "v2" }, { "date": "2026-10-12", "metric": "length-normalized weighted rubric score", "name": "HealthBench Professional", "score": 62.3, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "dataset": "Artificial Analysis AutomationBench-AA held-out private set (657 tasks)", "metric": "strict pass@1", "name": "AutomationBench", "score": 45.4, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis AutomationBench-AA", "version": "1.0.6" }, { "date": "2026-10-12", "metric": "score", "name": "GDP.pdf", "score": 33.2, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "date": "2026-10-12", "metric": "mean reward", "name": "JobBench", "score": 75.3, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "metric": "mean partial score", "name": "OSWorld", "score": 78.4, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "osworld-v2.1" }, { "metric": "binary task completion rate", "name": "OSWorld", "score": 39.4, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; source labels this binary completion rate", "version": "osworld-v2.1" }, { "harness": "mini-swe-agent", "metric": "accuracy", "name": "Terminal-Bench", "score": 57.1, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort; Artificial Analysis Terminal-Bench 4.0-AA", "version": "4.0.0" }, { "dataset": "DeepSWE v1.1 (113 tasks)", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 79.0, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "1.1" }, { "date": "2026-10-12", "metric": "pass@1", "name": "SWE-Marathon", "score": 57.5, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "xhigh effort" }, { "date": "2026-10-12", "harness": "mini-swe-agent", "metric": "share of rollouts passing at least 95% of tests", "name": "ProgramBench", "score": 59.0, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 56, "source": "https://research.meta.ai/static/muse-wonder-model-card", "variant": "max effort", "version": "4.3" }, { "cost": 10.528007, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0 (66 tasks); reported ±3.31 percentage points", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 58.081, "source": "https://www.vals.ai/models/meta_muse-wonder-full", "variant": "Vals AI independent run; source labels score accuracy; avg@3 (each run pass@1)", "version": "4.0.0" }, { "dataset": "Terminal-Bench Science v0.1.0 (70 tasks); reported ±5.82 percentage points", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 37.14, "source": "https://www.vals.ai/models/meta_muse-wonder-full", "variant": "Vals AI single pass@1 run; source labels score accuracy", "version": "0.1.0" }, { "dataset": "GDPval-AA v2.1 (220 tasks; 44 occupations; 9 industries); reported CI −25/+25 Elo", "effort": "Max", "harness": "Stirrup agent loop (shell and web browsing)", "metric": "Elo rating", "name": "GDPval-AA", "score": 1857, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 6.809627, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "Vals Index composite (finance, coding, legal, tax)", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 66.787, "source": "https://www.vals.ai/models/meta_muse-wonder-full", "variant": "model profile reports ±1.07", "version": "2.2" }, { "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 50.0772123627021, "source": "https://artificialanalysis.ai/models/muse-wonder-full", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 53.6, "source": "https://artificialanalysis.ai/models/muse-wonder-full", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 48.5, "source": "https://artificialanalysis.ai/models/muse-wonder-full", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Muse Wonder Full is the most capable model in Meta's Muse Wonder series, built for long-running professional agents and ultra-long-horizon software engineering. It shares Muse Wonder's Active Memory and Active Focus capabilities and generally scores higher on Meta's agentic and coding evaluations.", "family": "muse", "limit": { "context": 1048576 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://research.meta.ai/blog/muse-wonder" }, { "label": "Model card", "type": "model_card", "url": "https://research.meta.ai/static/muse-wonder-model-card" } ], "modalities": { "input": [ "text", "image", "video", "audio" ], "output": [ "text" ] }, "name": "Muse Wonder Full", "open_weights": false, "release_date": "2026-10-12", "schema": 2 }, "microsoft/mai-code-1-flash": { "benchmarks": [ { "date": "2026-06-02", "harness": "GitHub Copilot", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 51.2, "source": "https://microsoft.ai/news/introducingmai-code-1-flash/" }, { "metric": "resolved", "name": "SWE-Bench Verified", "score": 71.6, "source": "https://llm-stats.com/benchmarks/swe-bench-verified" }, { "metric": "success rate", "name": "Terminal-Bench", "score": 54.8, "source": "https://llm-stats.com/benchmarks/terminal-bench-2", "version": "2.0" }, { "metric": "accuracy", "name": "GPQA Diamond", "score": 84.6, "source": "https://llm-stats.com/benchmarks/gpqa" }, { "dataset": "Academic reasoning, text", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 18, "source": "https://microsoft.ai/pdf/MAI-Code-1-Flash-Model-Card.PDF" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Microsoft coding model built for fast, efficient assistance in everyday developer workflows", "family": "mai", "knowledge": "2025-12", "last_updated": "2026-06-08", "limit": { "context": 256000, "output": 128000 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://microsoft.ai/pdf/MAI-Code-1-Flash-Model-Card.PDF" }, { "label": "Announcement", "type": "announcement", "url": "https://microsoft.ai/news/introducingmai-code-1-flash/" } ], "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MAI-Code-1-Flash", "open_weights": false, "release_date": "2026-06-02", "schema": 2 }, "microsoft/mai-code-1.1-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Microsoft coding model with native vision support, optimized for fast and efficient software development", "family": "mai", "limit": { "context": 256000, "output": 128000 }, "links": [ { "label": "Announcement", "type": "announcement", "url": "https://microsoft.ai/news/mai-code-1-1-flash-br-better-faster-at-a-quarter-of-the-cost/" } ], "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "MAI-Code-1.1-Flash", "open_weights": false, "release_date": "2026-08-11", "schema": 2 }, "microsoft/mai-image-2.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and precise editing model", "family": "mai", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/png" ] }, "name": "MAI-Image-2.5", "open_weights": false, "release_date": "2026-06-02", "schema": 2, "type": "image", "v2_only": true }, "microsoft/mai-image-2.5-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Highest-fidelity MAI-Image-2.5 model for detailed editing and in-image text", "family": "mai", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/png" ] }, "name": "MAI-Image-2.5-Pro", "open_weights": false, "release_date": "2026-07-23", "schema": 2, "type": "image", "v2_only": true }, "microsoft/mai-image-2.6": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image generation and multi-reference editing model with web grounding", "family": "mai", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/png" ] }, "name": "MAI-Image-2.6", "open_weights": false, "release_date": "2026-08-14", "schema": 2, "type": "image", "v2_only": true }, "microsoft/mai-image-2.6-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost MAI-Image-2.6 for high-throughput generation and editing", "family": "mai", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "image/png" ] }, "name": "MAI-Image-2.6-Flash", "open_weights": false, "release_date": "2026-09-04", "schema": 2, "type": "image", "v2_only": true }, "microsoft/mai-transcribe-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual speech-to-text model across 43 languages", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "MAI-Transcribe-1.5", "open_weights": false, "release_date": "2026-06-02", "schema": 2, "type": "transcription", "v2_only": true }, "microsoft/mai-transcribe-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual speech-to-text model across 60 languages with language identification and code switching", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "MAI-Transcribe-2", "open_weights": false, "release_date": "2026-09-03", "schema": 2, "type": "transcription", "v2_only": true }, "microsoft/mai-voice-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Expressive multilingual text-to-speech model for long-form and conversational voice", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "MAI-Voice-2", "open_weights": false, "release_date": "2026-06-02", "schema": 2, "type": "speech", "v2_only": true }, "microsoft/mai-voice-2-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency MAI-Voice-2 text-to-speech model for voice agents", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "MAI-Voice-2-Flash", "open_weights": false, "release_date": "2026-07-23", "schema": 2, "type": "speech", "v2_only": true }, "microsoft/mai-voice-2.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Highest-fidelity MAI text-to-speech model across 23 languages", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "MAI-Voice-2.1", "open_weights": false, "release_date": "2026-10-01", "schema": 2, "type": "speech", "v2_only": true }, "microsoft/mai-voice-2.1-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency MAI-Voice-2.1 text-to-speech model", "family": "mai", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "MAI-Voice-2.1-Flash", "open_weights": false, "release_date": "2026-10-01", "schema": 2, "type": "speech", "v2_only": true }, "microsoft/phi-4-mini": { "benchmarks": [ { "metric": "accuracy", "name": "MMLU", "score": 67.3, "source": "https://huggingface.co/microsoft/Phi-4-mini-instruct/resolve/main/README.md" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Microsoft instruction model tuned for efficient coding assistance, reasoning, and low-latency agent tasks", "family": "phi", "knowledge": "2024-06", "limit": { "context": 128000, "output": 4096 }, "links": [ { "label": "Weights", "type": "weights", "url": "https://huggingface.co/microsoft/Phi-4-mini-instruct" } ], "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/microsoft/Phi-4-mini-instruct" ], "total_billion": 3.8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Phi-4-mini", "open_weights": true, "release_date": "2025-02-26", "schema": 2 }, "microsoft/phi-4-multimodal-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Microsoft multimodal model for text, image, and speech understanding", "family": "phi", "knowledge": "2024-06", "license": "MIT", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official full multimodal architecture count, including vision and speech encoders and adapters.", "sources": [ "https://huggingface.co/microsoft/Phi-4-multimodal-instruct" ], "total_billion": 5.6 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/mpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Phi-4-multimodal-instruct", "open_weights": true, "release_date": "2025-02-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/microsoft/Phi-4-multimodal-instruct" } ] }, "microsoft/wizardlm-2-8x22b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Microsoft WizardLM instruction-tuned Mixtral MoE for multilingual dialogue and complex tasks", "license": "Apache-2.0", "limit": { "context": 65536, "output": 8000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete unquantized checkpoint in the third-party re-upload already linked by this local entry; not an original Microsoft weight host.", "sources": [ "https://huggingface.co/api/models/alpindale/WizardLM-2-8x22B" ], "total_billion": 140.620634112 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "WizardLM 2 8x22B", "open_weights": true, "release_date": "2024-04-15", "schema": 2, "weights": [ { "label": "Hugging Face archive", "url": "https://huggingface.co/alpindale/WizardLM-2-8x22B" } ] }, "mindlab/macaron-v1-tall": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Qwen3.6-based Mixture-of-LoRA model with specialists for chat, personal agents, coding, and generative UI", "license": "MIT", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "L0 Chat LoRA specialist", "notes": "Author explicitly reports 3,775,651,840 stored values per adapter; L0 is BF16. Not a per-request active count.", "sources": [ "https://huggingface.co/mindlab-research/Macaron-V1-Tall" ], "total_billion": 3.77565184 }, { "name": "L1 Agent LoRA specialist", "notes": "Author-reported stored values per adapter; L1 is BF16.", "sources": [ "https://huggingface.co/mindlab-research/Macaron-V1-Tall" ], "total_billion": 3.77565184 }, { "name": "L2 Coding LoRA specialist", "notes": "Author-reported stored values per adapter; L2 is F32.", "sources": [ "https://huggingface.co/mindlab-research/Macaron-V1-Tall" ], "total_billion": 3.77565184 }, { "name": "L3 GenUI LoRA specialist", "notes": "Author-reported stored values per adapter; L3 is BF16.", "sources": [ "https://huggingface.co/mindlab-research/Macaron-V1-Tall" ], "total_billion": 3.77565184 } ], "notes": "The card's nominal release footprint is not treated as an inclusive exact total. Only independently disclosed specialist weight counts are recorded; they are not summed." } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Macaron V1 Tall", "open_weights": true, "release_date": "2026-07-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mindlab-research/Macaron-V1-Tall" } ] }, "mindlab/macaron-v1-venti": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "GLM-5.2-based Mixture-of-LoRA model with specialists for chat, personal agents, coding, and generative UI", "license": "MIT", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Author-reported full Mixture-of-LoRA release footprint, including the base and all four specialists; not a per-request active count.", "sources": [ "https://huggingface.co/mindlab-research/Macaron-V1-Venti" ], "total_billion": 748 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Macaron V1 Venti", "open_weights": true, "release_date": "2026-07-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mindlab-research/Macaron-V1-Venti" } ] }, "minimax/MiniMax-H3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "MiniMax general-purpose video model with multimodal references and synchronized stereo audio. Open H3-Base checkpoints; Context-IR and 2K regeneration are not included in the initial weights release.", "family": "minimax", "last_updated": "2026-10-04", "license": "MiniMax H3 Community License Agreement", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://www.minimax.io/blog/minimax-h3" }, { "type": "announcement", "url": "https://www.minimax.io/news/minimax-h3-open-source" }, { "type": "model_card", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3" }, { "type": "license", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "H3-Omni-Transformer", "notes": "Includes the AdaLN-related branches in the released transformer. Their precomputable inference outputs do not turn them into a per-token active count.", "sources": [ "https://huggingface.co/MiniMaxAI/MiniMax-H3" ], "total_billion": 33 } ], "notes": "Omni-transformer statistic only; separate H3-Encoder and visual/audio VAEs are not included in an inferred full-model total." } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4", "audio" ] }, "name": "MiniMax H3", "open_weights": true, "release_date": "2026-07-31", "schema": 2, "type": "video", "weights": [ { "format": "safetensors", "label": "Hugging Face: H3-Base FL2VA / Ref2VA", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3" } ] }, "minimax/MiniMax-H3-Highspeed": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "MiniMax H3 video tier that balances quality and speed, with multimodal references and synchronized audio", "family": "minimax", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://platform.minimax.io/docs/api-reference/video-generation-v2-create" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "MiniMax H3 Highspeed", "open_weights": false, "release_date": "2026-07-31", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-H3-Image": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "MiniMax text-to-image and image-editing model", "family": "minimax", "license": "MiniMax H3 Image Community License Agreement", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://www.minimax.io/blog/minimax-h3-image" }, { "type": "model_card", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image" }, { "type": "license", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image/blob/main/LICENSE" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "MiniMax H3 Image", "open_weights": true, "release_date": "2026-11-10", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "format": "safetensors", "label": "Hugging Face: MiniMax-H3-Image", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image" } ] }, "minimax/MiniMax-H3-Image-Highspeed": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster tier of MiniMax H3 Image for text-to-image and image editing", "family": "minimax", "license": "MiniMax H3 Image Community License Agreement", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://www.minimax.io/blog/minimax-h3-image" }, { "type": "model_card", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image" }, { "type": "license", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image/blob/main/LICENSE" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif" ], "output": [ "image/jpeg", "image/png", "image/webp" ] }, "name": "MiniMax H3 Image Highspeed", "open_weights": true, "release_date": "2026-11-10", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "format": "safetensors", "label": "Hugging Face: MiniMax-H3-Image", "url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image" } ] }, "minimax/MiniMax-H3-Lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lower-cost MiniMax H3 video tier for creative exploration and batch generation, with multimodal references", "family": "minimax", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://platform.minimax.io/docs/api-reference/video-generation-v2-create" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "MiniMax H3 Lite", "open_weights": false, "release_date": "2026-07-31", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-H3-Max": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast video model post-trained by fal.ai on MiniMax H3, with synchronized audio at 480p and 768p", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/heic", "image/heif", "video/mp4", "video/quicktime", "audio/wav", "audio/mpeg" ], "output": [ "video/mp4" ] }, "name": "MiniMax H3 Max", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-Hailuo-02": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for 512P to 1080P clips of up to 10 seconds from text, a first-frame image or first and last frames", "family": "minimax", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "announcement", "url": "https://www.minimax.io/news/minimax-hailuo-02" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Hailuo 02", "open_weights": false, "release_date": "2025-06-18", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-Hailuo-2.3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for realistic human motion, cinematic effects and stylized scenes from text or a first-frame image", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Hailuo 2.3", "open_weights": false, "release_date": "2025-10-28", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-Hailuo-2.3-Fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Faster, lower-cost Hailuo 2.3 variant that animates a first-frame image with a text prompt", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Hailuo 2.3 Fast", "open_weights": false, "release_date": "2025-10-28", "schema": 2, "type": "video", "v2_only": true }, "minimax/MiniMax-M2": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 69.4, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Efficient open MiniMax model built for coding agents and tool-heavy workflows", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "sources": [ "https://huggingface.co/MiniMaxAI/MiniMax-M2" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2", "open_weights": true, "release_date": "2025-10-27", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2" } ] }, "minimax/MiniMax-M2-Her": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "MiniMax M2 variant tuned for conversational and character-driven agent interactions", "family": "minimax", "limit": { "context": 65536, "output": 2048 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2 Her", "open_weights": false, "release_date": "2026-01-23", "schema": 2 }, "minimax/MiniMax-M2.1": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 74, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M2.1" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 36.81, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Earlier MiniMax agent model for practical coding and productivity tasks", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "notes": "Explicit total and active counts published by Artificial Analysis for M2.1, not inferred from another M2 version or quantized tensor storage.", "sources": [ "https://artificialanalysis.ai/models/minimax-m2-1" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2.1", "open_weights": true, "release_date": "2025-12-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.1" } ] }, "minimax/MiniMax-M2.5": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 75.8, "source": "https://www.swebench.com/" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 10.3, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 19.52, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 18.6, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Prior MiniMax coding model for agent workflows, office edits, and automation", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "notes": "Explicit total and active counts published by Artificial Analysis for M2.5, not inferred from another M2 version or quantized tensor storage.", "sources": [ "https://artificialanalysis.ai/models/minimax-m2-5" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2.5", "open_weights": true, "release_date": "2026-02-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5" } ] }, "minimax/MiniMax-M2.5-highspeed": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "High-speed MiniMax model for low-latency coding and agent workflows", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "notes": "Counts for the M2.5 checkpoint linked by this local speed-tier entry; no distinct architecture is assumed from the API speed tier.", "sources": [ "https://artificialanalysis.ai/models/minimax-m2-5", "https://huggingface.co/MiniMaxAI/MiniMax-M2.5" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2.5-highspeed", "open_weights": true, "release_date": "2026-02-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5" } ] }, "minimax/MiniMax-M2.7": { "benchmarks": [ { "date": "2026-06-01", "harness": "Claude Code", "metric": "resolved", "name": "SWE-Bench Verified", "score": 79.9, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "harness": "Claude Code", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 56.2, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "metric": "success rate", "name": "Terminal-Bench", "score": 51.1, "source": "https://www.minimax.io/blog/minimax-m3", "version": "2.1" }, { "dataset": "HLE no-tool subset", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 28.0, "source": "https://arxiv.org/html/2605.26494", "variant": "thinking enabled; no tools; single sample (pass@1); Artificial Analysis Index v4.0 protocol" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Open MiniMax flagship for coding agents, office automation, and complex environments", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "notes": "Explicit total and active counts published by Artificial Analysis for M2.7, not inferred from another M2 version or quantized tensor storage.", "sources": [ "https://artificialanalysis.ai/models/minimax-m2-7" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2.7", "open_weights": true, "release_date": "2026-03-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.7" } ] }, "minimax/MiniMax-M2.7-highspeed": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Low-latency M2.7 variant for interactive coding plans and agent loops", "family": "minimax", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10, "notes": "Counts for the M2.7 checkpoint linked by this local speed-tier entry; no distinct architecture is assumed from the API speed tier.", "sources": [ "https://artificialanalysis.ai/models/minimax-m2-7", "https://huggingface.co/MiniMaxAI/MiniMax-M2.7" ], "total_billion": 230 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax-M2.7-highspeed", "open_weights": true, "release_date": "2026-03-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.7" } ] }, "minimax/MiniMax-M3": { "benchmarks": [ { "date": "2026-06-01", "harness": "Claude Code", "metric": "resolved", "name": "SWE-Bench Verified", "score": 80.5, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "harness": "Claude Code", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 59.0, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "metric": "success rate", "name": "Terminal-Bench", "score": 66.0, "source": "https://www.minimax.io/blog/minimax-m3", "version": "2.1" }, { "date": "2026-06-01", "metric": "accuracy", "name": "BrowseComp", "score": 83.52, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "metric": "success rate", "name": "MCP Atlas", "score": 74.2, "source": "https://www.minimax.io/blog/minimax-m3" }, { "date": "2026-06-01", "metric": "success rate", "name": "OSWorld-Verified", "score": 70.06, "source": "https://www.minimax.io/blog/minimax-m3" }, { "cost": 3.0398771584794297, "cost_source": "https://artificialanalysis.ai/models/minimax-m3", "cost_unit": "USD/task", "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "2% (rounded display)", "source": "https://artificialanalysis.ai/models/comparisons/glm-5-3-flash-vs-minimax-m3", "variant": "pass@1 averaged over three repeats; Artificial Analysis page displays a rounded integer; exact unrounded score not exposed; 500-step cap", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": "0% (rounded display)", "source": "https://artificialanalysis.ai/models/comparisons/glm-5-3-flash-vs-minimax-m3", "variant": "pass@1 averaged over three repeats; Artificial Analysis page displays a rounded integer; exact unrounded score not exposed; 1,000-step cap", "version": "0.1.0" }, { "dataset": "May 2025 revision; 2,158 text-only questions", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 39, "source": "https://artificialanalysis.ai/models/comparisons/minimax-m3-vs-glm-4-6", "variant": "pass@1; no tools; GPT-5.6 Luna (medium) equality checker" }, { "cost": 1.2850974407373852, "cost_unit": "USD/weighted task", "dataset": "Nine constituent evaluations in Artificial Analysis Intelligence Index v5.0.1", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 18.8789083859255, "source": "https://artificialanalysis.ai/models/minimax-m3", "variant": "Reasoning version", "version": "5.0.1" }, { "cost": 7.229497, "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1.01, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "MiniMax multimodal model for long-context coding, perception, and agent planning", "family": "minimax", "limit": { "context": 1048576, "output": 512000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 23, "notes": "Both counts are approximate in the official model card.", "sources": [ "https://huggingface.co/MiniMaxAI/MiniMax-M3" ], "total_billion": 428 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "video/x-msvideo", "video/quicktime", "video/x-matroska" ], "output": [ "text" ] }, "name": "MiniMax-M3", "open_weights": true, "release_date": "2026-06-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3" } ] }, "minimax/MiniMax-M3.1-Flash": { "benchmarks": [ { "dataset": "v1.1 (113 tasks)", "harness": "mini-SWE-agent 2.4.5", "metric": "pass@1", "name": "DeepSWE", "score": 73.7, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "variant": "512K context", "version": "1.1" }, { "dataset": "Full public V2 split (642 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 96.57, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "Scale leaderboard alias: Minimax M3p1 Flash; reported ±1.16", "version": "2.0.0" }, { "dataset": "terminal-bench/terminal-bench@4.0.0 (66 tasks)", "harness": "Codex 0.149.1", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 38.1, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "version": "4.0.0" }, { "dataset": "harborframework/terminal-bench-science; terminalbenchscience_0_1 (70 tasks)", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 3.3, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash/blob/main/.eval_results/terminal-bench-science.yaml", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "600 publicly released tasks", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 62.8, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "version": "1.0.6" }, { "dataset": "Text-only subset (2,158 questions)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.4, "source": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "variant": "without tools", "version": "unspecified" }, { "cost": 0.659684, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "date": "2026-10-20", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 53.781, "source": "https://www.vals.ai/models/minimax_MiniMax-M3.1-flash", "variant": "Official MiniMax API; temperature=1; top_p=0.95; max output=262,144 tokens", "version": "2.2" }, { "cost": 0.49097067348453205, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 29.0595214290458, "source": "https://artificialanalysis.ai/models/minimax-m3-1-flash", "version": "5.0.1" }, { "cost": 0.734722, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 25.253, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Frontier multimodal coding model with 1M context window and tunable thinking depth", "family": "minimax", "limit": { "context": 1000000, "output": 512000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 23, "notes": "Approximate official architecture counts; verified against the now-published model card on 2026-11-06.", "sources": [ "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash" ], "total_billion": 428 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "video/x-msvideo", "video/quicktime", "video/x-matroska" ], "output": [ "text" ] }, "name": "MiniMax-M3.1-Flash", "open_weights": true, "release_date": "2026-10-19", "schema": 2 }, "minimax/MiniMax-M3.1-Flash-Preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Frontier multimodal coding model with 1M context window and tunable thinking depth", "family": "minimax", "limit": { "context": 1000000, "output": 512000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "video/x-msvideo", "video/quicktime", "video/x-matroska" ], "output": [ "text" ] }, "name": "MiniMax-M3.1-Flash-Preview", "open_weights": false, "release_date": "2026-09-27", "schema": 2 }, "minimax/MiniMax-Text-01": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "MiniMax model for chat, coding, office work, and agentic tasks", "family": "minimax", "limit": { "context": 1000000, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax Text 01 (MiniMax)", "open_weights": false, "release_date": "2025-01-15", "schema": 2 }, "minimax/image-01": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "MiniMax text-to-image generation model. The API also takes one character (portrait) reference to keep a person consistent, but it cannot edit an input image", "family": "minimax", "last_updated": "2026-08-25", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image/jpeg" ] }, "name": "MiniMax image-01", "open_weights": false, "release_date": "2025-02-15", "schema": 2, "type": "image" }, "minimax/image-01-live": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "MiniMax image generation model with style presets. The API also takes one character (portrait) reference to keep a person consistent, but it cannot edit an input image", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image/jpeg" ] }, "name": "MiniMax image-01-live", "open_weights": false, "release_date": "2025-02-15", "schema": 2, "type": "image", "v2_only": true }, "minimax/minimax-m1-80k": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Open hybrid-attention MoE reasoning model with a million-token context and an 80K thinking budget", "family": "minimax", "license": "Apache-2.0", "limit": { "context": 1000000, "output": 80000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 45.9, "sources": [ "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k" ], "total_billion": 456 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniMax M1 80K", "open_weights": true, "release_date": "2025-06-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/MiniMaxAI/MiniMax-M1-80k" } ] }, "minimax/speech-2.8-hd": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "High-fidelity text-to-speech model with sound tags and voice cloning", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/flac", "audio/wav", "audio/basic", "audio/opus" ] }, "name": "Speech 2.8 HD", "open_weights": false, "release_date": "2026-01-23", "schema": 2, "type": "speech", "v2_only": true }, "minimax/speech-2.8-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast text-to-speech model with sound tags and voice cloning", "family": "minimax", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm", "audio/flac", "audio/wav", "audio/basic", "audio/opus" ] }, "name": "Speech 2.8 Turbo", "open_weights": false, "release_date": "2026-01-23", "schema": 2, "type": "speech", "v2_only": true }, "mistral/codestral-22b-v0.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Mistral code model for fill-in-the-middle and 80+ programming languages", "family": "codestral", "license": "Mistral AI Non-Production License", "limit": { "context": 32768, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the Transformers checkpoint; alternate consolidated-format files are not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Codestral-22B-v0.1" ], "total_billion": 22.247282688 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Codestral-22B-v0.1", "open_weights": true, "release_date": "2024-05-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" } ] }, "mistral/codestral-2508": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral code model for low-latency completion, fill-in-the-middle, and code generation", "family": "codestral", "limit": { "context": 128000, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Codestral 25.08", "open_weights": false, "release_date": "2025-07-30", "schema": 2 }, "mistral/codestral-embed-2505": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Embedding model for code search, retrieval, and code-aware RAG pipelines", "family": "codestral-embed", "limit": { "context": 8192, "output": 3072 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Codestral Embed", "open_weights": false, "release_date": "2025-05-28", "schema": 2, "type": "embedding", "v2_only": true }, "mistral/codestral-latest": { "benchmarks": [ { "date": "2025-01-13", "metric": "percent correct", "name": "Aider Polyglot", "score": 11.1, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral code model for completions, refactors, and developer IDE workflows", "family": "codestral", "knowledge": "2024-10", "last_updated": "2025-01-04", "limit": { "context": 256000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "The Codestral-22B-v0.1 checkpoint linked by this local entry, not a guarantee for future API alias targets. Hub-published BF16 count; alternate formats not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Codestral-22B-v0.1" ], "total_billion": 22.247282688 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Codestral (latest)", "open_weights": true, "release_date": "2024-05-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" } ] }, "mistral/devstral-2512": { "benchmarks": [ { "date": "2026-05-31", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 23.7, "source": "https://openrouter.ai/mistralai/devstral-2512/benchmarks" }, { "date": "2026-05-31", "metric": "percent correct", "name": "SciCode", "score": 33.1, "source": "https://openrouter.ai/mistralai/devstral-2512/benchmarks" }, { "date": "2026-05-31", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 18.9, "source": "https://openrouter.ai/mistralai/devstral-2512/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral's coding-agent model for repository work, terminal tasks, and software fixes", "family": "devstral", "knowledge": "2025-12", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official documentation's advertised full-model parameter count, not inferred from FP8 tensor storage.", "sources": [ "https://docs.mistral.ai/models/devstral-2-25-12.md" ], "total_billion": 123 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Devstral 2", "open_weights": true, "release_date": "2025-12-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Devstral-2-123B-Instruct-2512" } ] }, "mistral/devstral-medium-2507": { "benchmarks": [ { "date": "2025-07-10", "metric": "resolved", "name": "SWE-Bench Verified", "score": 61.6, "source": "https://mistral.ai/news/devstral-2507" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral coding agent model for repository tasks and software engineering workflows", "family": "devstral", "knowledge": "2025-05", "limit": { "context": 128000, "output": 128000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Devstral Medium", "open_weights": false, "release_date": "2025-07-10", "schema": 2 }, "mistral/devstral-medium-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral coding agent model for repository tasks and software engineering workflows", "family": "devstral", "knowledge": "2025-12", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official documentation's parameter count for the Devstral 2 checkpoint linked by this local alias.", "sources": [ "https://docs.mistral.ai/models/devstral-2-25-12.md" ], "total_billion": 123 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Devstral 2 (latest)", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Devstral-2-123B-Instruct-2512" } ] }, "mistral/devstral-small-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multimodal coding model for repository exploration, file editing, and software agents", "family": "devstral", "knowledge": "2025-12", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512" ], "total_billion": 24 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Devstral Small 2", "open_weights": true, "release_date": "2025-12-09", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Devstral-Small-2-24B-Instruct-2512" } ] }, "mistral/devstral-small-2507": { "benchmarks": [ { "date": "2025-07-10", "metric": "resolved", "name": "SWE-Bench Verified", "score": 53.6, "source": "https://mistral.ai/news/devstral-2507" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral coding agent model for repository tasks and software engineering workflows", "family": "devstral", "knowledge": "2025-05", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Devstral-Small-2507" ], "total_billion": 24 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Devstral Small", "open_weights": true, "release_date": "2025-07-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Devstral-Small-2507" } ] }, "mistral/magistral-medium-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral reasoning model for transparent analysis, math, and complex decisions", "family": "magistral-medium", "knowledge": "2025-06", "last_updated": "2025-03-20", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Magistral Medium (latest)", "open_weights": false, "release_date": "2025-03-17", "schema": 2 }, "mistral/magistral-small-2506": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Mistral reasoning model for transparent step-by-step problem solving", "family": "magistral", "license": "Apache 2.0", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Magistral-Small-2506" ], "total_billion": 24 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Magistral Small", "open_weights": true, "release_date": "2025-06-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Magistral-Small-2506" } ] }, "mistral/magistral-small-2509": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open multimodal reasoning model for transparent analysis of text and images", "family": "magistral", "license": "Apache 2.0", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Magistral-Small-2509" ], "total_billion": 24 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Magistral Small 1.2", "open_weights": true, "release_date": "2025-09-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Magistral-Small-2509" } ] }, "mistral/ministral-14b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact multimodal Mistral model for local assistants, edge agents, and efficient tool use", "family": "ministral", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal count from Mistral's official BF16 release; its card explicitly links the no-loss FP8 checkpoint referenced by this local entry. Not a sum of rounded language and vision sizes.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-14B-Instruct-2512-BF16", "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512-BF16" ], "total_billion": 13.94503168 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 14B", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512" } ] }, "mistral/ministral-14b-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads", "family": "ministral", "limit": { "context": 262144, "output": 209715 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published complete multimodal count from Mistral's official BF16 release, whose card links the no-loss FP8 edition. Documentation maps the API name to this December 2025 model.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-14B-Instruct-2512-BF16", "https://docs.mistral.ai/models/ministral-3-14b-25-12.md" ], "total_billion": 13.94503168 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 14B 2512", "open_weights": true, "release_date": "2025-12-02", "schema": 2 }, "mistral/ministral-3-14b-instruct-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open vision-language model for efficient local deployment, instruction following, and tool use", "family": "ministral", "license": "Apache 2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal count from the official BF16 release, whose card explicitly links the no-loss FP8 edition; not just the 13.5B language component.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-14B-Instruct-2512-BF16", "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512-BF16" ], "total_billion": 13.94503168 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 14B", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512" } ] }, "mistral/ministral-3-3b-instruct-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact open vision-language model for edge deployment, instruction following, and tool use", "family": "ministral", "license": "Apache 2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal count from the official BF16 release, whose card explicitly links the no-loss FP8 edition; not the nominal 3B designation or a component sum.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-3B-Instruct-2512-BF16", "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512-BF16" ], "total_billion": 3.849090048 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 3B", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512" } ] }, "mistral/ministral-3-8b-instruct-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact open vision-language model for edge deployment, instruction following, and tool use", "family": "ministral", "license": "Apache 2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal count from the official BF16 release, whose card explicitly links the no-loss FP8 edition; not the nominal 8B designation or a component sum.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-8B-Instruct-2512-BF16", "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512-BF16" ], "total_billion": 8.91802624 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 8B", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512" } ] }, "mistral/ministral-3b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads", "family": "ministral", "knowledge": "2024-03", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "OpenRouter explicitly describes the October 2024 Ministral 3B as a 3B-parameter model. Mistral's documentation identifies the retired ministral-3b-2410 version and distinguishes its December 2025 replacement. This is a rounded published specification, not a count inherited from Ministral 3 3B Instruct 2512. Public availability of the 2024 checkpoint is not established by this parameter source.", "sources": [ "https://openrouter.ai/mistralai/ministral-3b", "https://docs.mistral.ai/models/ministral-3b-24-1.md" ], "total_billion": 3 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ministral 3B", "open_weights": true, "release_date": "2024-10-16", "schema": 2 }, "mistral/ministral-3b-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads", "family": "ministral", "limit": { "context": 131072, "output": 104857 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal BF16 checkpoint count; official documentation and OpenRouter identify this API entry as the December 2025 Ministral 3 3B Instruct model. Not the 2024 Ministral 3B model.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-3B-Instruct-2512-BF16", "https://docs.mistral.ai/models/ministral-3-3b-25-12.md" ], "total_billion": 3.849090048 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 3B 2512", "open_weights": true, "release_date": "2025-12-02", "schema": 2 }, "mistral/ministral-8b-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads", "family": "ministral", "limit": { "context": 262144, "output": 209715 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal BF16 checkpoint count; official documentation and OpenRouter identify this API entry as the December 2025 Ministral 3 8B Instruct model.", "sources": [ "https://huggingface.co/api/models/mistralai/Ministral-3-8B-Instruct-2512-BF16", "https://docs.mistral.ai/models/ministral-3-8b-25-12.md" ], "total_billion": 8.91802624 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Ministral 3 8B 2512", "open_weights": true, "release_date": "2025-12-02", "schema": 2 }, "mistral/ministral-8b-instruct-2410": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient open Mistral edge model for on-device chat and function calling", "family": "ministral", "license": "Mistral Research License", "limit": { "context": 131072, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Exact count of 8,019,808,256 explicitly reported in the model card, converted to billions.", "sources": [ "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410" ], "total_billion": 8.019808256 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Ministral 8B Instruct", "open_weights": true, "release_date": "2024-10-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410" } ] }, "mistral/mistral-7b-instruct-v0.3": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open 7B Mistral instruction model with function calling", "family": "mistral", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the Transformers checkpoint; alternate consolidated-format files are not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Mistral-7B-Instruct-v0.3" ], "total_billion": 7.248023552 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral 7B Instruct v0.3", "open_weights": true, "release_date": "2024-05-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3" } ] }, "mistral/mistral-embed": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Embedding model for semantic search, retrieval, clustering, and ranking pipelines", "family": "mistral-embed", "limit": { "context": 8000, "output": 3072 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Mistral Embed", "open_weights": false, "release_date": "2023-12-11", "schema": 2, "type": "embedding" }, "mistral/mistral-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Flagship Mistral model for advanced reasoning, coding, and multilingual work", "family": "mistral-large", "knowledge": "2024-11-30", "limit": { "context": 128000, "output": 102400 }, "modalities": { "input": [ "text", "application/pdf" ], "output": [ "text" ] }, "name": "Mistral Large", "open_weights": false, "release_date": "2024-02-26", "schema": 2 }, "mistral/mistral-large-2407": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Flagship Mistral model for advanced reasoning, coding, and multilingual work", "family": "mistral-large", "knowledge": "2024-03-31", "limit": { "context": 131072, "output": 104857 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407" ], "total_billion": 123 } } }, "modalities": { "input": [ "text", "application/pdf" ], "output": [ "text" ] }, "name": "Mistral Large 2407", "open_weights": true, "release_date": "2024-07-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407" } ] }, "mistral/mistral-large-2411": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 13.8, "source": "https://openrouter.ai/mistralai/mistral-large-2407/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 29.2, "source": "https://openrouter.ai/mistralai/mistral-large-2407/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 6.1, "source": "https://openrouter.ai/mistralai/mistral-large-2407/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Mistral model for advanced reasoning, coding, and multilingual work", "family": "mistral-large", "knowledge": "2024-11", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Large-Instruct-2411" ], "total_billion": 123 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral Large 2.1", "open_weights": true, "release_date": "2024-11-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2411" } ] }, "mistral/mistral-large-2512": { "benchmarks": [ { "date": "2026-06-04", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 22.7, "source": "https://openrouter.ai/mistralai/mistral-large-2512/benchmarks" }, { "date": "2026-06-04", "metric": "percent correct", "name": "SciCode", "score": 36.2, "source": "https://openrouter.ai/mistralai/mistral-large-2512/benchmarks" }, { "date": "2026-06-04", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 15.9, "source": "https://openrouter.ai/mistralai/mistral-large-2512/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral's largest general model for enterprise agents, coding, and multilingual reasoning", "family": "mistral-large", "knowledge": "2024-11", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 41, "notes": "Full multimodal model; the card separately reports 673B total and 39B active for the language component only.", "sources": [ "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512" ], "total_billion": 675 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Large 3", "open_weights": true, "release_date": "2025-12-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512" } ] }, "mistral/mistral-large-4": { "benchmarks": [ { "name": "DeepSWE", "score": 61.7, "source": "https://mistral.ai/news/mistral-large-4/", "variant": "public preview", "version": "1.1" }, { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 28.3, "source": "https://mistral.ai/news/mistral-large-4/", "variant": "public preview", "version": "4.0.0" }, { "cost": 5.180898, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 27.778, "source": "https://vals-ai.com/benchmarks/terminal-bench-4", "variant": "avg@3 (3 runs; pass@1 each)", "version": "4.0.0" }, { "cost": 1.4120973060945472, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 23.1277321742266, "source": "https://artificialanalysis.ai/models/mistral-large-4-medium", "version": "5.0.1" }, { "cost": 4.208903290646695, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 25.336451488165, "source": "https://artificialanalysis.ai/models/mistral-large-4-preview", "variant": "public preview", "version": "5.0.1" }, { "cost": 2.0617306268126856, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 26.1249946754784, "source": "https://artificialanalysis.ai/models/mistral-large-4", "version": "5.0.1" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "High", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 17.3, "source": "https://artificialanalysis.ai/models/mistral-large-4", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral's 1T-parameter multimodal MoE for reasoning, coding, and agentic work", "family": "mistral-large", "limit": { "context": 1048576, "output": 262144 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Large 4", "open_weights": false, "release_date": "2026-10-06", "schema": 2 }, "mistral/mistral-large-4-1t-a52b": { "benchmarks": [ { "cost": 3.716552, "cost_unit": "USD/test", "effort": "High", "metric": "performance score", "name": "Vals Index", "score": 56.527, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "version": "2.2" }, { "cost": 2.728364, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 26.263, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "tools": { "supported": true } }, "description": "October 29 open-weight release of Mistral Large 4, a multimodal MoE model with 1.05T total and 52B active parameters, improved through additional RL for coding, cybersecurity, and reasoning.", "family": "mistral-large", "license": "Mistral Ecosystem Open-Weight License", "limit": { "context": 1048576 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 52, "notes": "Approximate counts. Active count includes embeddings and output layers; the card's 49B figure excludes them. Total reported as ~1T.", "sources": [ "https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Mistral Large 4 (Le Chonk)", "open_weights": true, "release_date": "2026-10-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B" } ] }, "mistral/mistral-large-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Mistral model for advanced reasoning, coding, and multilingual work", "family": "mistral-large", "knowledge": "2024-11", "last_updated": "2025-12-02", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 41, "notes": "The checkpoint linked by this local alias. Full multimodal count, not the language-only 673B total / 39B active.", "sources": [ "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512" ], "total_billion": 675 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Large (latest)", "open_weights": true, "release_date": "2024-11-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512" } ] }, "mistral/mistral-medium-2505": { "benchmarks": [ { "date": "2026-05-30", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 13.6, "source": "https://openrouter.ai/mistralai/mistral-medium-3/benchmarks" }, { "date": "2026-05-30", "metric": "percent correct", "name": "SciCode", "score": 33.1, "source": "https://openrouter.ai/mistralai/mistral-medium-3/benchmarks" }, { "date": "2026-05-30", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 3.8, "source": "https://openrouter.ai/mistralai/mistral-medium-3/benchmarks" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 4.52, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral-medium", "knowledge": "2025-05", "limit": { "context": 131072, "output": 131072 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Medium 3", "open_weights": false, "release_date": "2025-05-07", "schema": 2 }, "mistral/mistral-medium-2604": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 77.6, "source": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" }, { "date": "2026-05-22", "metric": "accuracy", "name": "τ³-Telecom", "score": 91.4, "source": "https://mistral.ai/news/vibe-remote-agents-mistral-medium-3-5/", "variant": "public preview" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/step-5-vs-mistral-medium-3-5", "variant": "pass@1 averaged over three repeats", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/mistral-medium-3-5-vs-mistral-small-4", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "2,158 text-only questions from the May 2025 HLE revision (2,500 questions total)", "harness": "Artificial Analysis HLE evaluation", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 14, "source": "https://artificialanalysis.ai/models/comparisons/mistral-medium-3-5-vs-glm-5-1", "variant": "pass@1; GPT-5.6 Luna (medium) equality checker" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 5, "source": "https://artificialanalysis.ai/models/mistral-medium-3-5", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Mistral model for enterprise assistants, multilingual work, and tools", "family": "mistral-medium", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" ], "total_billion": 128 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Medium 3.5", "open_weights": true, "release_date": "2026-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" } ] }, "mistral/mistral-medium-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral-medium", "knowledge": "2025-03-31", "limit": { "context": 131072, "output": 104857 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "application/pdf" ], "output": [ "text" ] }, "name": "Mistral Medium 3", "open_weights": false, "release_date": "2025-05-07", "schema": 2 }, "mistral/mistral-medium-3-5": { "benchmarks": [ { "cost": 1.7048494182799918, "cost_source": "https://artificialanalysis.ai/models/mistral-medium-3-5", "cost_unit": "USD/task", "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/step-5-vs-mistral-medium-3-5", "variant": "pass@1 averaged over three repeats", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/mistral-medium-3-5-vs-mistral-small-4", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "2,158 text-only questions from the May 2025 HLE revision (2,500 questions total)", "harness": "Artificial Analysis HLE evaluation", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 14, "source": "https://artificialanalysis.ai/models/comparisons/mistral-medium-3-5-vs-glm-5-1", "variant": "pass@1; GPT-5.6 Luna (medium) equality checker" }, { "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 5, "source": "https://artificialanalysis.ai/models/mistral-medium-3-5", "version": "5.0.1" }, { "cost": 1.0390849688366068, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 4.64508756526957, "source": "https://artificialanalysis.ai/models/mistral-medium-3-5", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral-medium", "limit": { "context": 262144, "output": 209715 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "application/pdf" ], "output": [ "text" ] }, "name": "Mistral Medium 3.5", "open_weights": false, "release_date": "2026-04-30", "schema": 2 }, "mistral/mistral-medium-3.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral-medium", "knowledge": "2025-06-30", "limit": { "context": 131072, "output": 104857 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "application/pdf" ], "output": [ "text" ] }, "name": "Mistral Medium 3.1", "open_weights": false, "release_date": "2025-08-12", "schema": 2 }, "mistral/mistral-medium-latest": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 77.6, "source": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced Mistral model for enterprise assistants, multilingual work, and tools", "family": "mistral-medium", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "The checkpoint linked by this local alias; not a guarantee for future alias targets.", "sources": [ "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" ], "total_billion": 128 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Medium (latest)", "open_weights": true, "release_date": "2026-04-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B" } ] }, "mistral/mistral-moderation-2603": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Moderation classifier for text and conversations with jailbreak detection and multilingual coverage", "limit": { "context": 131072, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral Moderation 26.03", "open_weights": false, "release_date": "2026-03-01", "schema": 2, "v2_only": true }, "mistral/mistral-nemo": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Mistral-NVIDIA open model for multilingual chat and local deployment", "family": "mistral-nemo", "knowledge": "2024-07", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete Transformers checkpoint; alternate consolidated-format files are not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Mistral-Nemo-Instruct-2407" ], "total_billion": 12.2477824 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral Nemo", "open_weights": true, "release_date": "2024-07-18", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407" } ] }, "mistral/mistral-ocr-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Document OCR model that extracts interleaved text and images from documents", "limit": { "context": 16384, "output": 0, "pdf": { "max_pages": 1000 } }, "modalities": { "input": [ "image/jpeg", "image/png", "image/avif", "image/tiff", "image/gif", "image/heic", "image/heif", "image/bmp", "image/webp", "application/pdf", "text/csv", "text/plain", "text/xml", "text/troff" ], "output": [ "text", "image/jpeg" ] }, "name": "Mistral OCR 3", "open_weights": false, "release_date": "2025-12-18", "schema": 2, "v2_only": true }, "mistral/mistral-ocr-4-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Document OCR model with paragraph-level bounding boxes, block labels, and confidence scores", "limit": { "context": 16384, "output": 0, "pdf": { "max_pages": 1000 } }, "modalities": { "input": [ "image/jpeg", "image/png", "image/avif", "image/tiff", "image/gif", "image/heic", "image/heif", "image/bmp", "image/webp", "application/pdf", "text/csv", "text/plain", "text/xml", "text/troff" ], "output": [ "text", "image/jpeg" ] }, "name": "Mistral OCR 4.1", "open_weights": false, "release_date": "2026-07-16", "schema": 2, "v2_only": true }, "mistral/mistral-saba": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral", "knowledge": "2024-09-30", "limit": { "context": 32768, "output": 26214 }, "modalities": { "input": [ "text", "application/pdf" ], "output": [ "text" ] }, "name": "Saba", "open_weights": false, "release_date": "2025-02-17", "schema": 2 }, "mistral/mistral-small-24b-instruct-2501": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Efficient Mistral model for fast chat, extraction, and production assistants", "family": "mistral-small", "knowledge": "2023-10-31", "limit": { "context": 32768, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501" ], "total_billion": 24 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral Small 3", "open_weights": true, "release_date": "2025-01-30", "schema": 2 }, "mistral/mistral-small-2506": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Mistral model for fast chat, extraction, and production assistants", "family": "mistral-small", "knowledge": "2025-03", "limit": { "context": 128000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official documentation's rounded parameter count for the Small 3.2 June 2025 release.", "sources": [ "https://docs.mistral.ai/models/mistral-small-3-2-25-06.md" ], "total_billion": 24 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Small 3.2", "open_weights": true, "release_date": "2025-06-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506" } ] }, "mistral/mistral-small-2603": { "benchmarks": [ { "date": "2026-06-01", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 24.3, "source": "https://openrouter.ai/mistralai/mistral-small-2603/benchmarks" }, { "date": "2026-06-01", "metric": "percent correct", "name": "SciCode", "score": 38, "source": "https://openrouter.ai/mistralai/mistral-small-2603/benchmarks" }, { "date": "2026-06-01", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 17.4, "source": "https://openrouter.ai/mistralai/mistral-small-2603/benchmarks" }, { "dataset": "Original HLE 2,500-question set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 10, "source": "https://artificialanalysis.ai/models/comparisons/mistral-small-4-vs-gpt-5-codex", "variant": "Reasoning" }, { "dataset": "Original HLE 2,500-question set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 4, "source": "https://artificialanalysis.ai/models/comparisons/mistral-small-4-non-reasoning-vs-qwen3-5-397b-a17b", "variant": "Non-reasoning" }, { "cost": 0.03990856296103761, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 4.29716249600478, "source": "https://artificialanalysis.ai/models/mistral-small-4", "variant": "Reasoning", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Mistral production model for chat, extraction, and cost-sensitive agents", "family": "mistral-small", "knowledge": "2025-06", "limit": { "context": 256000, "output": 256000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 6.5, "sources": [ "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603" ], "total_billion": 119 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Small 4", "open_weights": true, "release_date": "2026-03-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603" } ] }, "mistral/mistral-small-3-1-24b-instruct-2503": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient multimodal model for instruction following, coding, reasoning, and function calling", "family": "mistral-small", "knowledge": "2024-06", "limit": { "context": 128000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503" ], "total_billion": 24 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Small 3.1 24B", "open_weights": true, "release_date": "2025-03-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503" } ] }, "mistral/mistral-small-3.1-24b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Efficient Mistral model for fast chat, extraction, and production assistants", "family": "mistral-small", "knowledge": "2023-10-31", "limit": { "context": 128000, "output": 102400 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503" ], "total_billion": 24 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Small 3.1 24B", "open_weights": true, "release_date": "2025-03-17", "schema": 2 }, "mistral/mistral-small-3.2-24b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Mistral model for fast chat, extraction, and production assistants", "family": "mistral-small", "knowledge": "2023-10-31", "limit": { "context": 256000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official documentation's rounded parameter count for the Small 3.2 June 2025 release.", "sources": [ "https://docs.mistral.ai/models/mistral-small-3-2-25-06.md" ], "total_billion": 24 } } }, "modalities": { "input": [ "image/png", "image/jpeg", "image/webp", "image/gif", "text" ], "output": [ "text" ] }, "name": "Mistral Small 3.2 24B", "open_weights": true, "release_date": "2025-06-20", "schema": 2 }, "mistral/mistral-small-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Mistral model for fast chat, extraction, and production assistants", "family": "mistral-small", "knowledge": "2025-06", "limit": { "context": 256000, "output": 256000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 6.5, "notes": "The checkpoint linked by this local alias; not a guarantee for future alias targets.", "sources": [ "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603" ], "total_billion": 119 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Mistral Small (latest)", "open_weights": true, "release_date": "2026-03-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603" } ] }, "mistral/mixtral-8x22b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "mistral", "knowledge": "2024-01-31", "limit": { "context": 65536, "output": 52428 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 39, "sources": [ "https://mistral.ai/news/mixtral-8x22b" ], "total_billion": 141 } } }, "modalities": { "input": [ "text", "application/pdf" ], "output": [ "text" ] }, "name": "Mixtral 8x22B Instruct", "open_weights": true, "release_date": "2024-04-17", "schema": 2 }, "mistral/mixtral-8x7b-instruct-v0.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open sparse mixture-of-experts Mistral instruction model", "family": "mistral", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12.9, "notes": "Total is the Hub-published BF16 count for the complete checkpoint; per-token active count is explicitly reported by Mistral, not calculated from expert counts.", "sources": [ "https://huggingface.co/api/models/mistralai/Mixtral-8x7B-Instruct-v0.1", "https://mistral.ai/news/mixtral-of-experts" ], "total_billion": 46.702792704 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mixtral 8x7B Instruct v0.1", "open_weights": true, "release_date": "2023-12-11", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" } ] }, "mistral/pixtral-12b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral vision-language model for image understanding and multimodal chat", "family": "pixtral", "knowledge": "2024-09", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Multimodal Decoder", "sources": [ "https://huggingface.co/mistralai/Pixtral-12B-2409" ], "total_billion": 12 }, { "name": "Vision Encoder", "sources": [ "https://huggingface.co/mistralai/Pixtral-12B-2409" ], "total_billion": 0.4 } ], "notes": "Official decoder and vision encoder counts are separate observations, not an inferred complete total." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Pixtral 12B", "open_weights": true, "release_date": "2024-09-11", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Pixtral-12B-2409" } ] }, "mistral/pixtral-large-2502": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral vision-language model for image understanding and multimodal chat", "family": "pixtral", "limit": { "context": 128000, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Pixtral Large (25.02)", "open_weights": false, "release_date": "2025-04-08", "schema": 2 }, "mistral/pixtral-large-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mistral's larger vision model for document-heavy image understanding and chat", "family": "pixtral", "knowledge": "2024-11", "last_updated": "2024-11-04", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official complete multimodal count, including the vision encoder; not the 123B decoder-only count.", "sources": [ "https://huggingface.co/mistralai/Pixtral-Large-Instruct-2411" ], "total_billion": 124 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Pixtral Large (latest)", "open_weights": true, "release_date": "2024-11-01", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Pixtral-Large-Instruct-2411" } ] }, "mistral/voxtral-mini-2602": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with timestamps, optimized for transcription only", "family": "voxtral", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/ogg", "audio/webm" ], "output": [ "text" ] }, "name": "Voxtral Mini Transcribe 2", "open_weights": false, "release_date": "2026-02-04", "schema": 2, "type": "transcription", "v2_only": true }, "mistral/voxtral-mini-3b-2507": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open audio-language model for speech transcription, audio understanding, and voice-driven tool use", "family": "voxtral", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete audio-language Transformers checkpoint, not just the nominal 3B language backbone; alternate consolidated files not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Voxtral-Mini-3B-2507" ], "total_billion": 4.676271104 } } }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "Voxtral Mini 3B 2507", "open_weights": true, "release_date": "2025-07-15", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Voxtral-Mini-3B-2507" } ] }, "mistral/voxtral-mini-transcribe-realtime-2602": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency streaming speech-to-text model for live transcription and voice agents", "family": "voxtral", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602" ], "total_billion": 4 } } }, "modalities": { "input": [ "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "Voxtral Mini Transcribe Realtime", "open_weights": true, "release_date": "2026-02-04", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602" } ] }, "mistral/voxtral-mini-tts-2603": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text-to-speech model with zero-shot voice cloning", "family": "voxtral", "license": "CC-BY-NC-4.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official complete TTS model size, not the separately reported 3.4B transformer decoder backbone.", "sources": [ "https://mistral.ai/news/voxtral-tts" ], "total_billion": 4 } } }, "modalities": { "input": [ "text" ], "output": [ "audio/pcm", "audio/wav", "audio/mpeg", "audio/flac", "audio/opus" ] }, "name": "Voxtral Mini TTS", "open_weights": true, "release_date": "2026-03-23", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Voxtral-4B-TTS-2603" } ] }, "mistral/voxtral-small-24b-2507": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open audio-language model for speech transcription, audio understanding, and voice-driven tool use", "family": "voxtral", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete audio-language Transformers checkpoint; alternate consolidated files not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Voxtral-Small-24B-2507" ], "total_billion": 24.26180096 } } }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "Voxtral Small 24B 2507", "open_weights": true, "release_date": "2025-07-15", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mistralai/Voxtral-Small-24B-2507" } ] }, "mistral/voxtral-small-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Instruct model with native audio input for speech understanding and tool use", "family": "voxtral", "limit": { "context": 32000, "output": 32000 }, "metadata": { "modelatlas": { "parameters": { "notes": "The July 2025 checkpoint identified by this local entry. Hub-published BF16/F32 count for the complete audio-language checkpoint; alternate formats not summed.", "sources": [ "https://huggingface.co/api/models/mistralai/Voxtral-Small-24B-2507" ], "total_billion": 24.26180096 } } }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "Voxtral Small (latest)", "open_weights": true, "release_date": "2025-07-15", "schema": 2 }, "mixedbread/mxbai-embed-large-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model for retrieval with Matryoshka and binary quantization support", "license": "Apache-2.0", "limit": { "context": 512, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "mxbai-embed-large-v1", "open_weights": true, "release_date": "2024-03-07", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1" } ] }, "mixedbread/toast-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Specialized search model for knowledge-intensive questions, multi-step retrieval, and evidence synthesis", "limit": { "context": 131000, "output": 4000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Toast 1", "open_weights": false, "release_date": "2026-08-13", "schema": 2 }, "moonshotai/kimi-k2": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Kimi model for long-context chat, coding, and agentic reasoning", "family": "kimi-k2", "knowledge": "2024-12-31", "limit": { "context": 131072, "output": 98304 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Original July 11, 2025 K2 release represented by this local entry. Official total reported as 1T.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2-Instruct" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kimi K2 0711", "open_weights": true, "release_date": "2025-07-11", "schema": 2 }, "moonshotai/kimi-k2-0905": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "September 2025 Kimi K2 instruction update with a 256K context and improved agentic coding", "family": "kimi-k2", "license": "Modified MIT", "limit": { "context": 262144, "output": 98304 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official total reported as 1T, converted to B.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kimi K2 Instruct 0905", "open_weights": true, "release_date": "2025-09-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905" } ] }, "moonshotai/kimi-k2-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Non-thinking Kimi K2 MoE instruction model for coding and tool-using agents", "family": "kimi-k2", "license": "Modified MIT", "limit": { "context": 131072, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official total reported as 1T, converted to B.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2-Instruct" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kimi K2 Instruct", "open_weights": true, "release_date": "2025-07-11", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct" } ] }, "moonshotai/kimi-k2-thinking": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 71.3, "source": "https://huggingface.co/moonshotai/Kimi-K2-Thinking" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Thinking Kimi model for slower research passes, planning, and hard technical questions", "family": "kimi-thinking", "knowledge": "2024-08", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official total reported as 1T, converted to B.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2-Thinking" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kimi K2 Thinking", "open_weights": true, "release_date": "2025-11-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking" } ] }, "moonshotai/kimi-k2-thinking-turbo": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Kimi reasoning model for long-horizon research, planning, and tool use", "family": "kimi-thinking", "knowledge": "2024-08", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "The checkpoint linked by this local speed-tier entry. Official total reported as 1T.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2-Thinking" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Kimi K2 Thinking Turbo", "open_weights": true, "release_date": "2025-11-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking" } ] }, "moonshotai/kimi-k2.5": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 70.8, "source": "https://www.swebench.com/" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 13.1, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 20.95, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 25.77, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 24.37, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Earlier Kimi frontier model for long-context agents, coding, and multimodal work", "family": "kimi-k2", "knowledge": "2025-01", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official architecture table's total, reported as 1T; the card also lists a 400M vision encoder separately. No extra component sum inferred.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2.5" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K2.5", "open_weights": true, "release_date": "2026-01-27", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2.5" } ] }, "moonshotai/kimi-k2.6": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 80.2, "source": "https://huggingface.co/moonshotai/Kimi-K2.6" }, { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 50.5, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 59.8, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 27.3, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 64.3, "source": "https://artificialanalysis.ai/agents/coding-agents", "version": "2.1" }, { "dataset": "HLE-Full", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34.7, "source": "https://huggingface.co/moonshotai/Kimi-K2.6/raw/main/README.md", "variant": "Thinking mode enabled; no tools; maximum generation length 98,304 tokens" }, { "dataset": "HLE-Full", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.0, "source": "https://huggingface.co/moonshotai/Kimi-K2.6/raw/main/README.md", "variant": "Thinking mode enabled; search, code-interpreter, and web-browsing tools; maximum generation length 262,144 tokens; per-step limit 49,152 tokens; retain only the most recent round of tool-related messages after context threshold" }, { "dataset": "Chartography full task set (100 tasks)", "harness": "Inspect AI Chartography harness; leaderboard configuration: 10 runs per task, Gemini 3.5 Flash judge", "name": "Chartography", "score": 12.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Thinking on" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal Kimi workhorse for agent loops, coding tasks, and visual context", "family": "kimi-k2", "knowledge": "2025-01", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official architecture table's total, reported as 1T; the card also lists a 400M vision encoder separately. No extra component sum inferred.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2.6" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K2.6", "open_weights": true, "release_date": "2026-04-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2.6" } ] }, "moonshotai/kimi-k2.7-code": { "benchmarks": [ { "date": "2026-06-12", "harness": "Kimi Code CLI", "name": "Kimi Code Bench", "score": 62.0, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code", "version": "v2" }, { "date": "2026-06-12", "harness": "Kimi Code CLI", "name": "Program Bench", "score": 53.6, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" }, { "date": "2026-06-12", "harness": "Kimi Code CLI", "name": "MLS Bench Lite", "score": 35.1, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" }, { "date": "2026-06-12", "harness": "Kimi Code CLI", "metric": "success rate", "name": "MCP Atlas", "score": 76.0, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" }, { "date": "2026-06-12", "harness": "Kimi Code CLI", "metric": "success rate", "name": "MCP Mark Verified", "score": 81.1, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" }, { "date": "2026-06-12", "harness": "Kimi Code CLI", "name": "Kimi Claw 24/7 Bench", "score": 46.9, "source": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" }, { "dataset": "113 tasks", "effort": "Maximum reasoning effort", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 31.0, "source": "https://arxiv.org/abs/2610.00890", "version": "1.1" }, { "dataset": "66 tasks", "effort": "Maximum reasoning effort", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 0.0, "source": "https://arxiv.org/abs/2610.00890", "version": "4.0.0" }, { "dataset": "full 100-task released set", "effort": "Max reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 0.0, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge", "version": "unspecified" }, { "cost": 2.815536243274336, "cost_unit": "USD/task", "dataset": "113 tasks", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 30.53097345132743, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking", "family": "kimi-k2", "knowledge": "2025-01", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "Official architecture table's total, reported as 1T; the card also lists a 400M vision encoder separately. No extra component sum inferred.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2.7-Code" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K2.7 Code", "open_weights": true, "release_date": "2026-06-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" } ] }, "moonshotai/kimi-k2.7-code-highspeed": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Lower-latency Kimi Code variant for interactive edits and coding-agent loops", "family": "kimi-k2", "knowledge": "2025-01", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "notes": "The checkpoint linked by this local speed-tier entry. Uses the official architecture table's 1T total and 32B active, without inferring component sums.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K2.7-Code" ], "total_billion": 1000 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K2.7 Code Highspeed", "open_weights": true, "release_date": "2026-06-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K2.7-Code" } ] }, "moonshotai/kimi-k2.8-preview": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Kimi coding model with more efficient thinking and up to 1M context, available through Kimi Code", "family": "kimi-k2", "limit": { "context": 1048576 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K2.8 Preview", "open_weights": false, "release_date": "2026-09-11", "schema": 2 }, "moonshotai/kimi-k3": { "benchmarks": [ { "date": "2026-07-16", "harness": "Kimi Code", "metric": "pass@1", "name": "DeepSWE", "score": 67.5, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort", "version": "1.1" }, { "date": "2026-07-16", "harness": "Kimi Code", "metric": "accuracy", "name": "Terminal-Bench", "score": 88.3, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort", "version": "2.1" }, { "date": "2026-07-16", "harness": "Kimi Code", "metric": "dominance score", "name": "FrontierSWE", "score": 81.2, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort" }, { "date": "2026-07-16", "harness": "Kimi Code", "metric": "score", "name": "Program Bench", "score": 77.8, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort" }, { "date": "2026-07-16", "harness": "Claude Code", "metric": "resolve rate", "name": "SWE Marathon", "score": 42.0, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort", "version": "1.1" }, { "date": "2026-07-16", "metric": "Elo", "name": "GDPval-AA", "score": 1668, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort", "version": "v2" }, { "date": "2026-07-16", "metric": "Elo", "name": "AA-Briefcase", "score": 1548, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort" }, { "dataset": "600-task public subset", "date": "2026-07-16", "metric": "success rate", "name": "AutomationBench", "score": 30.8, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort" }, { "date": "2026-07-16", "metric": "score", "name": "JobBench", "score": 52.9, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort" }, { "date": "2026-07-16", "harness": "Claude Code", "metric": "score", "name": "SpreadsheetBench", "score": 34.8, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort", "version": "2" }, { "date": "2026-07-16", "metric": "accuracy", "name": "BrowseComp", "score": 91.2, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort, context compaction" }, { "date": "2026-07-16", "metric": "accuracy", "name": "CharXiv Reasoning", "score": 91.3, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort, with tools" }, { "date": "2026-07-16", "metric": "pass@5", "name": "ZeroBench", "score": 41.0, "source": "https://www.kimi.com/blog/kimi-k3", "variant": "max effort, with tools" }, { "dataset": "V2 public set (642 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 97.7, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "effort": "Max", "harness": "mini-SWE-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.3, "source": "https://www.kimi.com/en/blog/kimi-k3", "version": "1.1" }, { "cost": 4.654682129933482, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 68.51441241685144, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 12.478176, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 17.172, "source": "https://vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; pass@1 per run; mean of three runs", "version": "4.0.0" }, { "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 1.43, "source": "https://vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1 single run; infrastructure/provider errors retried", "version": "0.1.0" }, { "dataset": "100-task Chartography set", "effort": "Max reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 26.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "20 trials per task; no tools" }, { "dataset": "HLE-Full", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.5, "source": "https://arxiv.org/pdf/2607.24653", "variant": "without tools" }, { "dataset": "HLE-Full", "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 56.0, "source": "https://arxiv.org/pdf/2607.24653", "variant": "with general tools" }, { "cost": 7.169465, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 52.955, "source": "https://vals.ai/models/kimi_kimi-k3", "version": "2.2" }, { "cost": 2.368773852349091, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 17.4191985300887, "source": "https://artificialanalysis.ai/models/kimi-k3-low", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work", "family": "kimi-k3", "limit": { "context": 1048576, "output": 1048576 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official total reported as 2.8T, converted to B. Active count not inferred from expert routing.", "sources": [ "https://www.kimi.com/blog/kimi-k3" ], "total_billion": 2800 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K3", "open_weights": true, "release_date": "2026-07-16", "schema": 2 }, "moonshotai/kimi-k3.1-pro": { "benchmarks": [ { "effort": "Max", "harness": "Claude-Code", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 65.7, "source": "https://www.kimi.com/en/blog/kimi-k3-1-pro", "variant": "Moonshot AI evaluation; temperature=1.0; top_p=0.95", "version": "4.0.0" }, { "dataset": "all 66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 67, "source": "https://artificialanalysis.ai/articles/kimi-k3-1-pro-launch-benchmarks", "variant": "pass@1 averaged over three repeats", "version": "4.0.0" }, { "cost": 4.56894, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "full 66-task set", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 63.636, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "mean of three pass@1 runs", "version": "4.0.0" }, { "dataset": "all 70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 34, "source": "https://artificialanalysis.ai/articles/artificial-analysis-intelligence-index-v5", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "all 70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 31.43, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1 single run", "version": "0.1.0" }, { "dataset": "full 100-task released set", "effort": "Max reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 16.7, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge", "version": "unspecified" }, { "dataset": "2,158 text-only questions from the 2,500-question May 2025 revision", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 52, "source": "https://artificialanalysis.ai/articles/kimi-k3-1-pro-launch-benchmarks", "variant": "pass@1", "version": "unspecified" }, { "cost": 6.440192, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "date": "2026-10-24", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 62.52, "source": "https://www.vals.ai/models/kimi_kimi-k3.1-pro", "version": "2.2" }, { "cost": 3.6812002895742055, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 46.4526236744189, "source": "https://artificialanalysis.ai/models/kimi-k3-1-pro", "version": "5.0.1" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 37.3, "source": "https://artificialanalysis.ai/models/kimi-k3-1-pro", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Multimodal Kimi model with 1M context and always-on reasoning for coding, 3D creation, financial research, and agentic work", "family": "kimi-k3", "license": "Kimi K3.1-Pro License", "limit": { "context": 1048576, "output": 1048576 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/moonshotai/Kimi-K3.1-Pro/blob/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e/README.md" }, { "label": "License", "type": "license", "url": "https://huggingface.co/moonshotai/Kimi-K3.1-Pro/blob/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 104, "components": [ { "name": "MoonViT-V2", "notes": "Vision encoder only; official 401M converted to B.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K3.1-Pro/blob/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e/README.md" ], "total_billion": 0.401 } ], "notes": "Official MoE architecture specifications: 2.8T total (converted to B) and 104B activated, not exact serialized tensor counts. The separately listed vision encoder is not added to the reported total.", "sources": [ "https://huggingface.co/moonshotai/Kimi-K3.1-Pro/blob/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e/README.md" ], "total_billion": 2800 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/heic", "image/heif", "video/mp4", "video/mpeg", "video/mov", "video/avi", "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/3gpp" ], "output": [ "text" ] }, "name": "Kimi K3.1 Pro", "open_weights": true, "release_date": "2026-10-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/moonshotai/Kimi-K3.1-Pro" } ] }, "morph/morph-v3-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "morph", "limit": { "context": 262144, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Morph V3 Fast", "open_weights": false, "release_date": "2025-07-07", "schema": 2 }, "morph/morph-v3-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Flagship model for demanding analysis, coding, and production agent workflows", "family": "morph", "limit": { "context": 262144, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Morph V3 Large", "open_weights": false, "release_date": "2025-07-07", "schema": 2 }, "morph/reflex": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Morph's 27B decision model that returns probabilities for typed choice, yes/no and score questions about text, JSON or images", "family": "morph", "limit": { "context": 65536, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "Reflex", "open_weights": false, "release_date": "2026-11-03", "schema": 2, "type": "decision", "v2_only": true }, "motif-technologies/motif-3": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 37.0, "source": "https://huggingface.co/Motif-Technologies/Motif-3", "variant": "temperature=1.0; top_p=0.95; max sequence length=262144" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Motif 3 is a large-scale, decoder-only Mixture-of-Experts (MoE) language model with 314 billion total parameters and 13.2 billion parameters activated per token.", "family": "motif", "license": "MIT", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 13.2, "notes": "Approximate counts as marked in the official architecture table.", "sources": [ "https://huggingface.co/Motif-Technologies/Motif-3" ], "total_billion": 314 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Motif 3", "open_weights": true, "release_date": "2026-08-07", "schema": 2, "weights": [ { "format": "safetensors", "label": "Model weights", "url": "https://huggingface.co/Motif-Technologies/Motif-3" } ] }, "nace-ai/drex-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "System One decision model for typed, calibrated probabilistic answers over text or structured state", "license": "Nace.AI Open RAIL-M", "limit": { "context": 139264, "output": 0 }, "links": [ { "label": "Documentation", "type": "docs", "url": "https://drex.nace.ai/docs" }, { "label": "License", "type": "license", "url": "https://huggingface.co/nace-ai/drex-v1.5/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Approximate parameter count explicitly reported in the model card.", "sources": [ "https://huggingface.co/nace-ai/drex-v1.5" ], "total_billion": 8.95 } } }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Drex 1.5", "open_weights": true, "release_date": "2026-09-28", "schema": 2, "type": "decision", "weights": [ { "format": "safetensors", "url": "https://huggingface.co/nace-ai/drex-v1.5" } ] }, "nace-ai/drex-v2.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "35B mixture-of-experts System One decision model with 3B active parameters, returning typed, calibrated probabilities over text or structured state without generating text", "last_updated": "2026-10-12", "license": "Nace.AI Open RAIL-M", "limit": { "context": 131072, "output": 0 }, "links": [ { "label": "Documentation", "type": "docs", "url": "https://github.com/nace-ai/drex-decision-models" }, { "label": "License", "type": "license", "url": "https://huggingface.co/nace-ai/drex-v2.0/blob/main/LICENSE" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint. No active count inherited from the backbone.", "sources": [ "https://huggingface.co/api/models/nace-ai/drex-v2.0" ], "total_billion": 35.107181936 } } }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Drex 2.0", "open_weights": true, "release_date": "2026-10-06", "schema": 2, "type": "decision", "weights": [ { "format": "safetensors", "url": "https://huggingface.co/nace-ai/drex-v2.0" } ] }, "nex-agi/nex-n2-pro": { "benchmarks": [ { "dataset": "May 2025 revision; 2,158 text-only questions", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34, "source": "https://artificialanalysis.ai/models/comparisons/nex-n2-pro-vs-gpt-5-5", "variant": "Reasoning version; pass@1; no tools" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open agentic MoE model (397B total, 17B active) for coding, tool use, and research workflows", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint. No active count inherited from the backbone.", "sources": [ "https://huggingface.co/api/models/nex-agi/Nex-N2-Pro" ], "total_billion": 396.802360816 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Nex-N2-Pro", "open_weights": true, "release_date": "2026-06-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nex-agi/Nex-N2-Pro" } ] }, "nomic-ai/nomic-embed-text-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Long-context open text embedding model for retrieval and semantic similarity", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Nomic Embed Text v1", "open_weights": true, "release_date": "2024-01-31", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nomic-ai/nomic-embed-text-v1" } ] }, "nomic-ai/nomic-embed-text-v1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Long-context open text embedding model with resizable output dimensions", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Nomic Embed Text v1.5", "open_weights": true, "release_date": "2024-02-10", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5" } ] }, "nousresearch/hermes-2-pro-llama-3-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Llama 3 fine-tune optimized for function calling, structured JSON responses, and dialogue", "family": "hermes", "license": "Llama 3 Community License", "limit": { "context": 8192, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 count for the complete main checkpoint used by the card's direct Transformers example; the separately hosted DPO adapter is not summed.", "sources": [ "https://huggingface.co/api/models/NousResearch/Hermes-2-Pro-Llama-3-8B", "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B" ], "total_billion": 8.030523392 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hermes 2 Pro Llama 3 8B", "open_weights": true, "release_date": "2024-04-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B" } ] }, "nousresearch/hermes-3-llama-3.1-405b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "nousresearch", "knowledge": "2023-12-31", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for this complete fine-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/NousResearch/Hermes-3-Llama-3.1-405B" ], "total_billion": 405.8533888 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hermes 3 405B Instruct", "open_weights": true, "release_date": "2024-08-16", "schema": 2 }, "nousresearch/hermes-3-llama-3.1-70b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "nousresearch", "knowledge": "2023-12-31", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for this complete fine-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/NousResearch/Hermes-3-Llama-3.1-70B" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hermes 3 70B Instruct", "open_weights": true, "release_date": "2024-08-18", "schema": 2 }, "nousresearch/hermes-4-405b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "family": "hermes", "knowledge": "2024-08-31", "limit": { "context": 131072, "output": 117964 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for this complete fine-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/NousResearch/Hermes-4-405B" ], "total_billion": 405.8533888 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hermes 4 405B", "open_weights": true, "release_date": "2025-08-26", "schema": 2 }, "nvidia/llama-3.1-nemotron-70b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron model for efficient reasoning, coding, and specialized AI agents", "family": "nemotron", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count from NVIDIA's own Transformers conversion, explicitly linked by the original model card.", "sources": [ "https://huggingface.co/api/models/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.1 Nemotron 70B Instruct", "open_weights": true, "release_date": "2025-04-15", "schema": 2 }, "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "nemotron", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3" ], "total_billion": 8.03 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.1 Nemotron Safety Guard 8B v3", "open_weights": true, "release_date": "2025-10-28", "schema": 2 }, "nvidia/llama-3.1-nemotron-ultra-253b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Nemotron model for high-throughput reasoning and complex agents", "family": "nemotron", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1" ], "total_billion": 253 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.1 Nemotron Ultra 253B", "open_weights": true, "release_date": "2025-04-07", "schema": 2 }, "nvidia/llama-3.3-nemotron-super-49b-v1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron model for efficient reasoning, coding, and specialized AI agents", "family": "nemotron", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/nvidia/Llama-3_3-Nemotron-Super-49B-v1" ], "total_billion": 49.867145216 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.3 Nemotron Super 49B v1", "open_weights": true, "release_date": "2025-03-18", "schema": 2 }, "nvidia/llama-3.3-nemotron-super-49b-v1.5": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron model for efficient reasoning, coding, and specialized AI agents", "family": "nemotron", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5" ], "total_billion": 49.867145216 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.3 Nemotron Super 49B v1.5", "open_weights": true, "release_date": "2025-07-25", "schema": 2 }, "nvidia/llama-nemotron-embed-1b-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text embedding model for retrieval and question answering pipelines", "family": "nemotron", "license": "nvidia-open-model-license", "limit": { "context": 8192, "output": 2048 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/nvidia/llama-nemotron-embed-1b-v2" ], "total_billion": 1.2358144 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Llama Nemotron Embed 1B v2", "open_weights": true, "release_date": "2025-10-16", "schema": 2, "type": "embedding", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2" } ] }, "nvidia/llama-nemotron-embed-vl-1b-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Embedding model for semantic search, retrieval, clustering, and ranking pipelines", "family": "nemotron", "limit": { "context": 10240, "output": 2048 }, "metadata": { "modelatlas": { "parameters": { "notes": "Approximately 1.7B for the complete vision-language encoder, not just the Llama language component.", "sources": [ "https://huggingface.co/nvidia/llama-nemotron-embed-vl-1b-v2" ], "total_billion": 1.7 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "embedding" ] }, "name": "Llama Nemotron Embed VL 1B v2", "open_weights": true, "release_date": "2025-12-18", "schema": 2, "type": "embedding" }, "nvidia/llama-nemotron-rerank-vl-1b-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Reranking model for improving retrieval quality in search and recommendation systems", "family": "nemotron", "limit": { "context": 10240, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Approximately 1.7B for the complete vision-language cross-encoder.", "sources": [ "https://huggingface.co/nvidia/llama-nemotron-rerank-vl-1b-v2" ], "total_billion": 1.7 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "image/bmp", "image/tiff" ], "output": [ "reranking" ] }, "name": "Llama Nemotron Rerank VL 1B v2", "open_weights": true, "release_date": "2025-12-18", "schema": 2, "type": "reranking" }, "nvidia/mistral-nemotron": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Mistral model for multilingual chat, reasoning, and tool-assisted workflows", "family": "nemotron", "last_updated": "2025-06-12", "limit": { "context": 128000, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Mistral Nemotron", "open_weights": false, "release_date": "2025-06-11", "schema": 2 }, "nvidia/nemotron-3-content-safety": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "nemotron", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/Nemotron-3-Content-Safety" ], "total_billion": 4 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3 Content Safety", "open_weights": true, "release_date": "2026-03-16", "schema": 2 }, "nvidia/nemotron-3-embed-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open text embedding model for retrieval, RAG, and code search", "family": "nemotron", "limit": { "context": 32768, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Approximate count explicitly reported in the official model card.", "sources": [ "https://huggingface.co/nvidia/Nemotron-3-Embed-1B-BF16" ], "total_billion": 1.14 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Nemotron 3 Embed 1B", "open_weights": true, "release_date": "2026-07-16", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/Nemotron-3-Embed-1B-BF16" } ] }, "nvidia/nemotron-3-embed-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual text embedding model for retrieval and semantic similarity", "family": "nemotron", "license": "OpenMDW-1.1", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Nemotron 3 Embed 8B", "open_weights": true, "release_date": "2026-07-16", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/Nemotron-3-Embed-8B-BF16" } ] }, "nvidia/nemotron-3-nano-30b-a3b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Small Nemotron 3 MoE for efficient coding, math, and long-context agents", "family": "nemotron", "knowledge": "2025-06-25", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.5, "notes": "Uses the card's explicit 3.5B active count, not the A3B model-name designation.", "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" ], "total_billion": 30 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3 Nano 30B A3B", "open_weights": true, "release_date": "2025-12-15", "schema": 2 }, "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": { "benchmarks": [ { "dataset": "220 tasks across 44 occupations and 9 major industries", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 237, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Nemotron omni model combining reasoning with text, vision, and audio", "family": "nemotron", "limit": { "context": 256000, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete multimodal checkpoint, not the 31B language backbone described separately in the card.", "sources": [ "https://huggingface.co/api/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16" ], "total_billion": 33.015632214 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "Nemotron 3 Nano Omni 30B A3B Reasoning", "open_weights": true, "release_date": "2026-04-28", "schema": 2 }, "nvidia/nemotron-3-super-120b-a12b": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 18.26, "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "variant": "no tools" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 22.82, "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "variant": "with tools" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron middle tier for collaborative agents and high-volume reasoning workloads", "family": "nemotron", "knowledge": "2025-06", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" ], "total_billion": 120 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3 Super 120B A12B", "open_weights": true, "release_date": "2026-03-11", "schema": 2 }, "nvidia/nemotron-3-ultra-550b-a55b": { "benchmarks": [ { "date": "2026-06-04", "metric": "resolved", "name": "SWE-Bench Verified", "score": 70.7, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" }, { "date": "2026-06-04", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 67.7, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" }, { "date": "2026-06-04", "metric": "success rate", "name": "Terminal-Bench", "score": 56.4, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "version": "2.1" }, { "date": "2026-06-04", "metric": "accuracy", "name": "GPQA", "score": 87.0, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "variant": "no tools" }, { "date": "2026-06-04", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 26.7, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "variant": "no tools" }, { "date": "2026-06-04", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 37.4, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "variant": "with tools" }, { "date": "2026-06-04", "metric": "pass@1", "name": "LiveCodeBench", "score": 89.0, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "version": "v6" }, { "date": "2026-06-04", "metric": "accuracy", "name": "MMLU-Pro", "score": 86.8, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" }, { "date": "2026-06-04", "metric": "accuracy", "name": "BrowseComp", "score": 44.4, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" }, { "date": "2026-06-04", "metric": "accuracy", "name": "IFBench", "score": 81.7, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "variant": "prompt loose" }, { "date": "2026-06-04", "metric": "wins or ties", "name": "GDPval", "score": 46.7, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" }, { "cost": 2.2257365504312134, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 11.7592726308177, "source": "https://artificialanalysis.ai/models/nvidia-nemotron-3-ultra-550b-a55b", "variant": "Reasoning", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "harness": "Stirrup agentic loop (shell and web browsing)", "metric": "Elo rating", "name": "GDPval-AA", "score": 1021, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Reasoning; CI -17/+17", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy", "family": "nemotron", "knowledge": "2025-09", "limit": { "context": 1000000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 55, "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" ], "total_billion": 550 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3 Ultra 550B A55B", "open_weights": true, "release_date": "2026-06-04", "schema": 2 }, "nvidia/nemotron-3.5-asr-streaming-multilingual-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open streaming speech recognition model for 40 language-locales", "family": "nemotron", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official multilingual Nemotron 3.5 streaming ASR card explicitly reports 600M parameters.", "sources": [ "https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b" ], "total_billion": 0.6 } } }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Nemotron 3.5 ASR Streaming Multilingual 0.6B", "open_weights": true, "release_date": "2026-08", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b" } ] }, "nvidia/nemotron-3.5-content-safety": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "nemotron", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/Nemotron-3.5-Content-Safety" ], "total_billion": 4 } } }, "modalities": { "input": [ "text", "image/gif", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Nemotron 3.5 Content Safety", "open_weights": true, "release_date": "2026-06-02", "schema": 2 }, "nvidia/nemotron-3.5-embed-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multimodal multilingual embedding model for text and document-image retrieval", "family": "nemotron", "license": "OpenMDW-1.1", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "embedding" ] }, "name": "Nemotron 3.5 Embed 1B", "open_weights": true, "release_date": "2026-10-20", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/Nemotron-3.5-Embed-1B-BF16" } ] }, "nvidia/nemotron-3.5-embed-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multimodal multilingual embedding model for text and document-image retrieval", "family": "nemotron", "license": "OpenMDW-1.1", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "embedding" ] }, "name": "Nemotron 3.5 Embed 8B", "open_weights": true, "release_date": "2026-10-19", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/Nemotron-3.5-Embed-8B-BF16" } ] }, "nvidia/nemotron-3.5-lightning": { "benchmarks": [ { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "MMLU-Pro", "score": 81.94, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "AA-Omniscience", "score": 17.5, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "SciCode", "score": 32.6, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "PinchBench", "score": 85.37, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "BrowseComp", "score": 36.97, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Gym / NeMo Evaluator SDK", "metric": "score", "name": "AA-LCR", "score": 52, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "metric": "accuracy", "name": "GPQA Diamond", "score": 75.44, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning; no tools" }, { "dataset": "text-only", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 11.72, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning; no tools" }, { "harness": "NeMo Evaluator", "metric": "accuracy", "name": "SWE-Bench Verified", "score": 51.56, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Evaluator", "metric": "accuracy", "name": "SWE-Bench Multilingual", "score": 39.33, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning" }, { "harness": "NeMo Evaluator", "metric": "accuracy", "name": "Terminal-Bench", "score": 24.58, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning", "version": "2.1" }, { "metric": "score", "name": "tau3-bench", "score": 9.28, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning; banking" }, { "metric": "Elo", "name": "GDPval-AA", "score": 832, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning", "version": "2" }, { "metric": "accuracy", "name": "IFBench", "score": 71.88, "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "variant": "BF16; reasoning; loose" }, { "cost": 0.28980699398265014, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 5.5346663903388, "source": "https://artificialanalysis.ai/models/nemotron-3-5-lightning", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast NVIDIA Nemotron MoE for reliable agentic tasks across enterprise workloads", "family": "nemotron", "knowledge": "2025-09", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16" ], "total_billion": 30 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3.5 Lightning 30B A3B", "open_weights": true, "release_date": "2026-08-11", "schema": 2 }, "nvidia/nemotron-3.5-super-120b-a12b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "NVIDIA Nemotron 3.5 Super reasoning model (120B total, 12B active) served with text-only input", "family": "nemotron", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "notes": "NVIDIA's official Nemotron page explicitly describes Nemotron 3.5 Super as a 120B-parameter MoE with 12B active per token. Perplexity independently identifies its exact perplexity/nemotron-3.5-super-120b-a12b entry as Nemotron 3.5 Super with text-only input. These are the named model's published architecture counts, not a measured endpoint inventory or evidence of a distinct pruned text-only checkpoint. No VL encoder size is subtracted.", "sources": [ "https://developer.nvidia.com/nemotron", "https://docs.perplexity.ai/docs/agent-api/models.md" ], "total_billion": 120 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron 3.5 Super 120B A12B", "open_weights": true, "release_date": "2026-10-15", "schema": 2 }, "nvidia/nemotron-3.5-super-vl-120b-a12b": { "benchmarks": [ { "dataset": "HLE full set, 2,500 questions, 1 sample per question", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 29.2, "source": "https://build.nvidia.com/nvidia/nemotron-3.5-super-vl-120b-a12b/modelcard", "variant": "BF16", "version": "unspecified" }, { "dataset": "HLE full set, 2,500 questions, 1 sample per question", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 29.0, "source": "https://build.nvidia.com/nvidia/nemotron-3.5-super-vl-120b-a12b/modelcard", "variant": "NVFP4", "version": "unspecified" }, { "cost": 0.862751, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0, 66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "Vals AI Baseten-hosted evaluation; reported ±0.00", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/nemotron-3-5-super-vl-vs-gpt-6-1-sol", "variant": "pass@1 averaged over 3 repeats; AA comparison displays rounded integer percent", "version": "0.1.0" }, { "metric": "performance score", "name": "Vals Index", "score": 29.76, "source": "https://www.vals.ai/models/baseten_nemotron-3.5-super-vl-120b-a12b", "variant": "Baseten-hosted model page; tested checkpoint precision not specified", "version": "2.2" }, { "cost": 0.5196327287868884, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 13.6203422723141, "source": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 30.7, "source": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 8.8, "source": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "NVIDIA vision-language reasoning model for text, image, and video understanding and agentic tool use", "family": "nemotron", "limit": { "context": 1048576, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16" ], "total_billion": 120 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4" ], "output": [ "text" ] }, "name": "Nemotron 3.5 Super VL 120B A12B", "open_weights": true, "release_date": "2026-10-15", "schema": 2 }, "nvidia/nemotron-cascade-2-30b-a3b": { "benchmarks": [ { "dataset": "HLE text-only split (2,158 examples)", "harness": "default system prompt with boxed-answer instruction; GPT-OSS-120B for answer extraction and correctness verification", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 17.7, "source": "https://arxiv.org/html/2603.19220", "variant": "no tool; thinking mode; 128K-token thinking budget" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron model for efficient reasoning, coding, and specialized AI agents", "family": "nemotron", "last_updated": "2026-04-09", "limit": { "context": 256000, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/nvidia/Nemotron-Cascade-2-30B-A3B" ], "total_billion": 30 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron Cascade 2 30B A3B", "open_weights": true, "release_date": "2026-03-19", "schema": 2 }, "nvidia/nemotron-content-safety-reasoning-4b": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "nemotron", "limit": { "context": 128000, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/Nemotron-Content-Safety-Reasoning-4B" ], "total_billion": 4 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron Content Safety Reasoning 4B", "open_weights": true, "release_date": "2026-01-22", "schema": 2 }, "nvidia/nemotron-mini-4b-instruct": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Nemotron model for efficient reasoning and deployable AI agents", "family": "nemotron", "last_updated": "2024-08-26", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "notes": "4,190,509,056 BF16 weight/bias values independently counted from all 324 tensor descriptors in this checkpoint's ZIP data.pkl. No weight payloads were loaded; the alternate NeMo serialization is not added. Not inferred from the nominal 4B designation.", "sources": [ "https://huggingface.co/nvidia/Nemotron-Mini-4B-Instruct/blob/791833e92ebddb0bc2c1007f6d2b6764f886a2ae/pytorch_model.bin" ], "total_billion": 4.190509056 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron Mini 4B Instruct", "open_weights": true, "release_date": "2024-08-21", "schema": 2 }, "nvidia/nemotron-nano-12b-v2-vl": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron multimodal model for visual reasoning and agentic AI workflows", "family": "nemotron", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16" ], "total_billion": 12.6 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "video/mp4", "video/x-matroska", "video/x-flv", "video/3gpp" ], "output": [ "text" ] }, "name": "Nemotron Nano 12B v2 VL", "open_weights": true, "release_date": "2025-10-28", "schema": 2 }, "nvidia/nemotron-nano-9b-v2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact Nemotron model for efficient reasoning and deployable AI agents", "family": "nemotron", "knowledge": "2024-09", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/nvidia/NVIDIA-Nemotron-Nano-9B-v2" ], "total_billion": 8.888227328 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Nemotron Nano 9B v2", "open_weights": true, "release_date": "2025-08-18", "schema": 2 }, "nvidia/nemotron-voicechat": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Nemotron multimodal model for visual reasoning and agentic AI workflows", "family": "nemotron", "limit": { "context": 128000, "output": 8192 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://build.nvidia.com/nvidia/nemotron-voicechat/modelcard" ], "total_billion": 12 } } }, "modalities": { "input": [ "text", "audio/wav" ], "output": [ "text" ] }, "name": "Nemotron VoiceChat", "open_weights": true, "release_date": "2026-03-16", "schema": 2 }, "nvidia/parakeet-tdt-0.6b-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open multilingual speech-to-text model with automatic language detection across 25 European languages", "family": "parakeet", "license": "CC-BY-4.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3" ], "total_billion": 0.6 } } }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg", "audio/webm", "audio/aac" ], "output": [ "text" ] }, "name": "Parakeet TDT 0.6B v3", "open_weights": true, "release_date": "2025-08", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3" } ] }, "openai/gpt-3.5-turbo": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 10.7, "source": "https://openrouter.ai/openai/gpt-3.5-turbo/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Compact GPT model for low-latency assistance and high-volume workloads", "family": "gpt", "knowledge": "2021-09-01", "last_updated": "2023-11-06", "limit": { "context": 16385, "output": 4096 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT-3.5-turbo", "open_weights": false, "release_date": "2023-03-01", "schema": 2 }, "openai/gpt-4": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 13.1, "source": "https://openrouter.ai/openai/gpt-4/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "GPT model for general reasoning, writing, coding, and tool-assisted tasks", "family": "gpt", "knowledge": "2023-11", "last_updated": "2024-04-09", "limit": { "context": 8192, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT-4", "open_weights": false, "release_date": "2023-11-06", "schema": 2 }, "openai/gpt-4-turbo": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 21.5, "source": "https://openrouter.ai/openai/gpt-4-turbo/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 31.9, "source": "https://openrouter.ai/openai/gpt-4-turbo/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Compact GPT model for low-latency assistance and high-volume workloads", "family": "gpt", "knowledge": "2023-12", "last_updated": "2024-04-09", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-4 Turbo", "open_weights": false, "release_date": "2023-11-06", "schema": 2 }, "openai/gpt-4.1": { "benchmarks": [ { "date": "2025-04-14", "metric": "percent correct", "name": "Aider Polyglot", "score": 52.4, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 5.4, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Long-lived GPT workhorse for coding, instruction following, and production apps", "family": "gpt", "knowledge": "2024-04", "limit": { "context": 1047576, "output": 32768 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-4.1", "open_weights": false, "release_date": "2025-04-14", "schema": 2 }, "openai/gpt-4.1-mini": { "benchmarks": [ { "date": "2025-04-14", "metric": "percent correct", "name": "Aider Polyglot", "score": 32.4, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Affordable GPT-4.1 lane for fast coding help and structured extraction", "family": "gpt-mini", "knowledge": "2024-04", "limit": { "context": 1047576, "output": 32768 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-4.1 mini", "open_weights": false, "release_date": "2025-04-14", "schema": 2 }, "openai/gpt-4.1-nano": { "benchmarks": [ { "date": "2025-04-14", "metric": "percent correct", "name": "Aider Polyglot", "score": 8.9, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Tiny GPT-4.1 option for classification, routing, and very high-volume tasks", "family": "gpt-nano", "knowledge": "2024-04", "limit": { "context": 1047576, "output": 32768 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-4.1 nano", "open_weights": false, "release_date": "2025-04-14", "schema": 2 }, "openai/gpt-4o": { "benchmarks": [ { "date": "2024-12-30", "metric": "percent correct", "name": "Aider Polyglot", "score": 23.1, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Omni-era GPT for multimodal chat, practical coding, and general assistants", "family": "gpt", "knowledge": "2023-09", "last_updated": "2024-08-06", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-4o", "open_weights": false, "release_date": "2024-05-13", "schema": 2 }, "openai/gpt-4o-2024-05-13": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 24.2, "source": "https://openrouter.ai/openai/gpt-4o-2024-05-13/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 30.9, "source": "https://openrouter.ai/openai/gpt-4o-2024-05-13/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "GPT model for general reasoning, writing, coding, and tool-assisted tasks", "family": "gpt", "knowledge": "2023-09", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-4o (2024-05-13)", "open_weights": false, "release_date": "2024-05-13", "schema": 2 }, "openai/gpt-4o-2024-08-06": { "benchmarks": [ { "date": "2024-12-30", "metric": "percent correct", "name": "Aider Polyglot", "score": 23.1, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 16.6, "source": "https://openrouter.ai/openai/gpt-4o-2024-08-06/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 33.1, "source": "https://openrouter.ai/openai/gpt-4o-2024-08-06/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 8.3, "source": "https://openrouter.ai/openai/gpt-4o-2024-08-06/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "GPT model for general reasoning, writing, coding, and tool-assisted tasks", "family": "gpt", "knowledge": "2023-09", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-4o (2024-08-06)", "open_weights": false, "release_date": "2024-08-06", "schema": 2 }, "openai/gpt-4o-2024-11-20": { "benchmarks": [ { "date": "2024-12-30", "metric": "percent correct", "name": "Aider Polyglot", "score": 18.2, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 16.7, "source": "https://openrouter.ai/openai/gpt-4o-2024-11-20/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 33.3, "source": "https://openrouter.ai/openai/gpt-4o-2024-11-20/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 8.3, "source": "https://openrouter.ai/openai/gpt-4o-2024-11-20/benchmarks" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 2.72, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "GPT model for general reasoning, writing, coding, and tool-assisted tasks", "family": "gpt", "knowledge": "2023-09", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-4o (2024-11-20)", "open_weights": false, "release_date": "2024-11-20", "schema": 2 }, "openai/gpt-4o-mini": { "benchmarks": [ { "date": "2024-12-21", "metric": "percent correct", "name": "Aider Polyglot", "score": 3.6, "source": "https://aider.chat/docs/leaderboards/" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 22.9, "source": "https://openrouter.ai/openai/gpt-4o-mini/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Small omni GPT for cheap multimodal assistance and production-scale traffic", "family": "gpt-mini", "knowledge": "2023-09", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-4o mini", "open_weights": false, "release_date": "2024-07-18", "schema": 2 }, "openai/gpt-4o-mini-transcribe": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model powered by GPT-4o Mini for fast, accurate transcription", "family": "gpt-mini", "knowledge": "2024-06-01", "limit": { "context": 16000, "output": 2000 }, "modalities": { "input": [ "text", "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm", "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "GPT-4o Mini Transcribe", "open_weights": false, "release_date": "2025-03-20", "schema": 2, "type": "transcription", "v2_only": true }, "openai/gpt-4o-mini-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-speech model powered by GPT-4o Mini with steerable voice delivery", "family": "gpt-mini", "limit": { "context": 2000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/opus", "audio/aac", "audio/flac", "audio/wav", "audio/pcm" ] }, "name": "GPT-4o Mini TTS", "open_weights": false, "release_date": "2025-03-20", "schema": 2, "type": "speech", "v2_only": true }, "openai/gpt-4o-transcribe": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model powered by GPT-4o for accurate transcription", "family": "gpt", "knowledge": "2024-06-01", "limit": { "context": 16000, "output": 2000 }, "modalities": { "input": [ "text", "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm", "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "GPT-4o Transcribe", "open_weights": false, "release_date": "2025-03-20", "schema": 2, "type": "transcription", "v2_only": true }, "openai/gpt-4o-transcribe-diarize": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with built-in speaker diarization", "family": "gpt", "knowledge": "2024-06-01", "limit": { "context": 16000, "output": 2000 }, "modalities": { "input": [ "text", "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm" ], "output": [ "text" ] }, "name": "GPT-4o Transcribe Diarize", "open_weights": false, "release_date": "2025-10-17", "schema": 2, "type": "transcription", "v2_only": true }, "openai/gpt-5": { "benchmarks": [ { "date": "2025-08-23", "metric": "percent correct", "name": "Aider Polyglot", "score": 88.0, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 41.78, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 25.32, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Original GPT-5 workhorse for reasoning, coding, writing, and tool workflows", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5", "open_weights": false, "release_date": "2025-08-07", "schema": 2 }, "openai/gpt-5-chat-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Chat-tuned GPT model for conversational assistance, writing, and tool workflows", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5 Chat (latest)", "open_weights": false, "release_date": "2025-08-07", "schema": 2 }, "openai/gpt-5-codex": { "benchmarks": [ { "date": "2026-06-01", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 38.9, "source": "https://openrouter.ai/openai/gpt-5-codex/benchmarks" }, { "date": "2026-06-01", "metric": "percent correct", "name": "SciCode", "score": 40.9, "source": "https://openrouter.ai/openai/gpt-5-codex/benchmarks" }, { "date": "2026-06-01", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 37.9, "source": "https://openrouter.ai/openai/gpt-5-codex/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Coding-optimized GPT model for repository edits, reviews, and agentic software work", "family": "gpt-codex", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5-Codex", "open_weights": false, "release_date": "2025-09-15", "schema": 2 }, "openai/gpt-5-image": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt", "knowledge": "2024-10-01", "limit": { "context": 400000, "output": 128000 }, "modalities": { "input": [ "image/jpeg", "image/png", "image/gif", "image/webp", "text", "application/pdf" ], "output": [ "image/png", "image/jpeg", "image/webp", "text" ] }, "name": "GPT-5 Image", "open_weights": false, "release_date": "2025-10-14", "schema": 2, "type": "image" }, "openai/gpt-5-image-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt", "limit": { "context": 400000, "output": 128000 }, "modalities": { "input": [ "application/pdf", "image/jpeg", "image/png", "image/gif", "image/webp", "text" ], "output": [ "image/png", "image/jpeg", "image/webp", "text" ] }, "name": "GPT-5 Image Mini", "open_weights": false, "release_date": "2025-10-16", "schema": 2, "type": "image" }, "openai/gpt-5-mini": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 19.44, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Small GPT-5 for responsive agents, coding help, and everyday automation", "family": "gpt-mini", "knowledge": "2024-05-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5 Mini", "open_weights": false, "release_date": "2025-08-07", "schema": 2 }, "openai/gpt-5-nano": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Tiny GPT-5 lane for routing, extraction, classification, and bulk jobs", "family": "gpt-nano", "knowledge": "2024-05-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5 Nano", "open_weights": false, "release_date": "2025-08-07", "schema": 2 }, "openai/gpt-5-pro": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 31.64, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning", "family": "gpt-pro", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 272000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5 Pro", "open_weights": false, "release_date": "2025-10-06", "schema": 2 }, "openai/gpt-5-search-api": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "GPT-5 model trained for web search in the Chat Completions API", "family": "gpt", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT-5 Search API", "open_weights": false, "release_date": "2025-10-14", "schema": 2, "type": "chat", "v2_only": true }, "openai/gpt-5.1": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 23.68, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "thinking; Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Sharper GPT-5 generation for coding, product work, and tool-assisted tasks", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.1", "open_weights": false, "release_date": "2025-11-13", "schema": 2 }, "openai/gpt-5.1-chat-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Chat-tuned GPT-5.1 for polished assistants, writing, and product conversations", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.1 Chat", "open_weights": false, "release_date": "2025-11-13", "schema": 2 }, "openai/gpt-5.1-codex": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Codex GPT for repository edits, code review, and practical software agents", "family": "gpt-codex", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.1 Codex", "open_weights": false, "release_date": "2025-11-13", "schema": 2 }, "openai/gpt-5.1-codex-max": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Coding-optimized GPT model for repository edits, reviews, and agentic software work", "family": "gpt-codex", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.1 Codex Max", "open_weights": false, "release_date": "2025-11-13", "schema": 2 }, "openai/gpt-5.1-codex-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Coding-optimized GPT model for repository edits, reviews, and agentic software work", "family": "gpt-codex", "knowledge": "2024-09-30", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.1 Codex mini", "open_weights": false, "release_date": "2025-11-13", "schema": 2 }, "openai/gpt-5.2": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 29.94, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 27.8, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Reliable GPT generation for broad coding, writing, and tool-assisted product work", "family": "gpt", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.2", "open_weights": false, "release_date": "2025-12-11", "schema": 2 }, "openai/gpt-5.2-chat-latest": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Chat-tuned GPT model for conversational assistance, writing, and tool workflows", "family": "gpt", "knowledge": "2025-08-31", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.2 Chat", "open_weights": false, "release_date": "2025-12-11", "schema": 2 }, "openai/gpt-5.2-codex": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 41.04, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Code-specialist GPT for repository edits, reviews, and long-running software agents", "family": "gpt-codex", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.2 Codex", "open_weights": false, "release_date": "2025-12-11", "schema": 2 }, "openai/gpt-5.2-pro": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Higher-accuracy GPT-5.2 variant for tougher reasoning and review workflows", "family": "gpt-pro", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.2 Pro", "open_weights": false, "release_date": "2025-12-11", "schema": 2 }, "openai/gpt-5.3-chat-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Chat-tuned GPT model for conversational assistance, writing, and tool workflows", "family": "gpt", "knowledge": "2025-08-31", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.3 Chat (latest)", "open_weights": false, "release_date": "2026-03-03", "schema": 2 }, "openai/gpt-5.3-codex": { "benchmarks": [ { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 32.6, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 42.38, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 38.98, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Coding-optimized GPT model for repository edits, reviews, and agentic software work", "family": "gpt-codex", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.3 Codex", "open_weights": false, "release_date": "2026-02-05", "schema": 2 }, "openai/gpt-5.3-codex-spark": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Coding-optimized GPT model for repository edits, reviews, and agentic software work", "family": "gpt-codex-spark", "knowledge": "2025-08-31", "limit": { "context": 128000, "input": 100000, "output": 32000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.3 Codex Spark", "open_weights": false, "release_date": "2026-02-05", "schema": 2 }, "openai/gpt-5.4": { "benchmarks": [ { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 59.1, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 40.8, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 36.3, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 44.29, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Codex CLI", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 44.36, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 40, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Codex", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 53.6, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Codex", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 72.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Codex", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 18.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Codex", "metric": "pass@1", "name": "Terminal-Bench", "score": 69.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "harness": "Cursor CLI", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 52.2, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 72.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 18.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "Terminal-Bench", "score": 64.7, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "date": "2026-04-23", "metric": "success rate", "name": "Terminal-Bench", "score": 75.1, "source": "https://openai.com/index/introducing-gpt-5-5/", "version": "2.0" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GPQA Diamond", "score": 92.8, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 39.8, "source": "https://openai.com/index/introducing-gpt-5-4/", "variant": "no tools" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 52.1, "source": "https://openai.com/index/introducing-gpt-5-4/", "variant": "with tools" }, { "date": "2026-04-23", "metric": "success rate", "name": "OSWorld-Verified", "score": 75.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "BrowseComp", "score": 82.7, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "wins or ties", "name": "GDPval", "score": 83.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "ARC-AGI-2", "score": 73.3, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "Verified" }, { "dataset": "Tier 1-3", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 47.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Tier 4", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 27.1, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "MMMU Pro", "score": 81.2, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "no tools" }, { "cost": 5.6524577245575225, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 51.76991150442478, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 runs", "version": "1.1" }, { "dataset": "surgeai/chartography", "effort": "xHigh", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 29.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs; Gemini 3.5 Flash judge", "version": "unspecified" }, { "dataset": "Original HLE set (2,500 questions)", "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44, "source": "https://artificialanalysis.ai/models/comparisons/gpt-5-4-vs-gpt-5" }, { "dataset": "220 tasks", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1247, "source": "https://artificialanalysis.ai/models/comparisons/gpt-5-4-vs-gpt-5", "version": "2.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "xhigh", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 36.24, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Agent-ready GPT for coding and computer-use workflows at a lower cost", "family": "gpt", "knowledge": "2025-08-31", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.4", "open_weights": false, "release_date": "2026-03-05", "schema": 2 }, "openai/gpt-5.4-image-2": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt", "limit": { "context": 272000, "output": 128000 }, "modalities": { "input": [ "image/jpeg", "image/png", "image/gif", "image/webp", "text", "application/pdf" ], "output": [ "image/png", "image/jpeg", "image/webp", "text" ] }, "name": "GPT-5.4 Image 2", "open_weights": false, "release_date": "2026-04-21", "schema": 2, "type": "image" }, "openai/gpt-5.4-mini": { "benchmarks": [ { "date": "2026-03-17", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 54.4, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Terminal-Bench", "score": 60.0, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh", "version": "2.0" }, { "date": "2026-03-17", "metric": "score", "name": "MCP Atlas", "score": 57.7, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "score", "name": "Toolathlon", "score": 42.9, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "τ²-Bench Telecom", "score": 93.4, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "GPQA Diamond", "score": 88.0, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 41.5, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "with tools" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 28.2, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "without tools" }, { "dataset": "GDPval: 220 tasks across 44 occupations and 9 industries", "harness": "Stirrup shell and web browsing; blind pairwise comparisons aggregated as Elo", "metric": "Elo rating", "name": "GDPval-AA", "score": 597, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Non-reasoning", "version": "2.1" }, { "date": "2026-03-17", "metric": "success rate", "name": "OSWorld-Verified", "score": 72.1, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "MMMU Pro", "score": 78.0, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "with Python" }, { "date": "2026-03-17", "metric": "accuracy", "name": "MMMU Pro", "score": 76.6, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "without tools" }, { "date": "2026-03-17", "metric": "overall edit distance", "name": "OmniDocBench", "score": 0.1263, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort none", "version": "1.5" }, { "date": "2026-03-17", "metric": "accuracy", "name": "OpenAI MRCR", "score": 47.7, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "8-needle, 64K-128K", "version": "v2" }, { "date": "2026-03-17", "metric": "accuracy", "name": "OpenAI MRCR", "score": 33.6, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "8-needle, 128K-256K", "version": "v2" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Graphwalks", "score": 76.3, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "BFS, 0-128K" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Graphwalks", "score": 71.5, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "parents, 0-128K" }, { "cost": 1.438224, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 2.525, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Strong small GPT for coding subagents, quick tool use, and high-volume work", "family": "gpt-mini", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.4 mini", "open_weights": false, "release_date": "2026-03-17", "schema": 2 }, "openai/gpt-5.4-nano": { "benchmarks": [ { "date": "2026-03-17", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 52.4, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Terminal-Bench", "score": 46.3, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh", "version": "2.0" }, { "date": "2026-03-17", "metric": "score", "name": "MCP Atlas", "score": 56.1, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "score", "name": "Toolathlon", "score": 35.5, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "τ²-Bench Telecom", "score": 92.5, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "GPQA Diamond", "score": 82.8, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 37.7, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "with tools" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 24.3, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "without tools" }, { "date": "2026-03-17", "metric": "success rate", "name": "OSWorld-Verified", "score": 39.0, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort xhigh" }, { "date": "2026-03-17", "metric": "accuracy", "name": "MMMU Pro", "score": 69.5, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "with Python" }, { "date": "2026-03-17", "metric": "accuracy", "name": "MMMU Pro", "score": 66.1, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "without tools" }, { "date": "2026-03-17", "metric": "overall edit distance", "name": "OmniDocBench", "score": 0.2419, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "reasoning effort none", "version": "1.5" }, { "date": "2026-03-17", "metric": "accuracy", "name": "OpenAI MRCR", "score": 44.2, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "8-needle, 64K-128K", "version": "v2" }, { "date": "2026-03-17", "metric": "accuracy", "name": "OpenAI MRCR", "score": 33.1, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "8-needle, 128K-256K", "version": "v2" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Graphwalks", "score": 73.4, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "BFS, 0-128K" }, { "date": "2026-03-17", "metric": "accuracy", "name": "Graphwalks", "score": 50.8, "source": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "variant": "parents, 0-128K" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation", "family": "gpt-nano", "knowledge": "2025-08-31", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.4 nano", "open_weights": false, "release_date": "2026-03-17", "schema": 2 }, "openai/gpt-5.4-pro": { "benchmarks": [ { "date": "2026-04-23", "metric": "accuracy", "name": "GPQA Diamond", "score": 94.4, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42.7, "source": "https://openai.com/index/introducing-gpt-5-4/", "variant": "no tools" }, { "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 58.7, "source": "https://openai.com/index/introducing-gpt-5-4/", "variant": "with tools" }, { "date": "2026-04-23", "metric": "accuracy", "name": "BrowseComp", "score": 89.3, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "wins or ties", "name": "GDPval", "score": 82.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Tier 1-3", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 50.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Tier 4", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 38.0, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "ARC-AGI-1", "score": 94.5, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "Verified" }, { "date": "2026-04-23", "metric": "accuracy", "name": "ARC-AGI-2", "score": 83.3, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "Verified" }, { "date": "2026-04-23", "metric": "accuracy", "name": "FinanceAgent", "score": 61.5, "source": "https://openai.com/index/introducing-gpt-5-5/", "version": "1.1" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GeneBench", "score": 25.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.32, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "More exact GPT-5.4 tier for demanding professional reasoning and agent tasks", "family": "gpt-pro", "knowledge": "2025-08-31", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.4 Pro", "open_weights": false, "release_date": "2026-03-05", "schema": 2 }, "openai/gpt-5.5": { "benchmarks": [ { "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 58.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-05-28", "harness": "Terminus-2", "metric": "success rate", "name": "Terminal-Bench", "score": 78.2, "source": "https://www.anthropic.com/news/claude-opus-4-8", "version": "2.1" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 45.43, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 44.79, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Codex", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 42.59, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" }, { "harness": "Codex", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 65.3, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "xhigh" }, { "harness": "Codex", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 80.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "xhigh" }, { "dataset": "hard-aa", "harness": "Codex", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 30.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "xhigh" }, { "harness": "Codex", "metric": "pass@1", "name": "Terminal-Bench", "score": 84.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "xhigh", "version": "2.1" }, { "harness": "Codex", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 60.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Codex", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 79.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Codex", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 26.2, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Codex", "metric": "pass@1", "name": "Terminal-Bench", "score": 75.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "harness": "Cursor CLI", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 57.8, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 75, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "dataset": "hard-aa", "harness": "Cursor CLI", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 24.9, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium" }, { "harness": "Cursor CLI", "metric": "pass@1", "name": "Terminal-Bench", "score": 73.4, "source": "https://artificialanalysis.ai/agents/coding-agents", "variant": "medium", "version": "2.1" }, { "date": "2026-04-23", "metric": "success rate", "name": "Terminal-Bench", "score": 82.7, "source": "https://openai.com/index/introducing-gpt-5-5/", "version": "2.0" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GPQA Diamond", "score": 93.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 41.4, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "no tools" }, { "date": "2026-04-23", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 52.2, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "with tools" }, { "date": "2026-04-23", "metric": "success rate", "name": "OSWorld-Verified", "score": 78.7, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "BrowseComp", "score": 84.4, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "MMMU Pro", "score": 81.2, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "no tools" }, { "date": "2026-04-23", "metric": "accuracy", "name": "ARC-AGI-2", "score": 85.0, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "Verified" }, { "dataset": "Tier 1-3", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 51.7, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Tier 4", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 35.4, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "wins or ties", "name": "GDPval", "score": 84.9, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "success rate", "name": "MCP Atlas", "score": 75.3, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "success rate", "name": "Toolathlon", "score": 55.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "success rate", "name": "τ²-Bench Telecom", "score": 98.0, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "original prompts" }, { "cost": 7.226236674778761, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.03539823008849, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "100 tasks", "effort": "xHigh", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 31, "source": "https://surgehq.ai/benchmarks/chartography" }, { "cost": 5.100437004424779, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 64.38053097345131, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 2.749342734513274, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.98230088495575, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 1.2002025442477877, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 26.991150442477874, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Default frontier GPT for coding, computer use, research, and knowledge work", "family": "gpt", "knowledge": "2025-12-01", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.5", "open_weights": false, "release_date": "2026-04-23", "schema": 2 }, "openai/gpt-5.5-instant": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Compact GPT model for low-latency assistance and high-volume workloads", "knowledge": "2025-12-01", "last_updated": "2026-05-28", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.5 Instant", "open_weights": false, "release_date": "2026-05-05", "schema": 2 }, "openai/gpt-5.5-pro": { "benchmarks": [ { "date": "2026-04-23", "metric": "accuracy", "name": "BrowseComp", "score": 90.1, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.1, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "no tools" }, { "date": "2026-04-23", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 57.2, "source": "https://openai.com/index/introducing-gpt-5-5/", "variant": "with tools" }, { "dataset": "Tier 1-3", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 52.4, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "dataset": "Tier 4", "date": "2026-04-23", "metric": "accuracy", "name": "FrontierMath", "score": 39.6, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "wins or ties", "name": "GDPval", "score": 82.3, "source": "https://openai.com/index/introducing-gpt-5-5/" }, { "date": "2026-04-23", "metric": "accuracy", "name": "GeneBench", "score": 33.2, "source": "https://openai.com/index/introducing-gpt-5-5/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Highest-accuracy GPT-5.5 tier for slower, precision-heavy reasoning and coding", "family": "gpt-pro", "knowledge": "2025-12-01", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.5 Pro", "open_weights": false, "release_date": "2026-04-23", "schema": 2 }, "openai/gpt-5.6-cyber": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Cybersecurity model for authorized vulnerability research and security testing", "family": "gpt", "knowledge": "2026-02-16", "limit": { "context": 400000, "input": 272000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-5.6 Cyber", "open_weights": false, "release_date": "2026-08-07", "schema": 2 }, "openai/gpt-5.6-luna": { "benchmarks": [ { "date": "2026-07-09", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 62.7, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Terminal-Bench", "score": 84.7, "source": "https://openai.com/index/gpt-5-6/", "version": "2.1" }, { "date": "2026-07-09", "metric": "resolve rate", "name": "DeepSWE", "score": 67.2, "source": "https://openai.com/index/gpt-5-6/", "version": "1.1" }, { "date": "2026-07-09", "metric": "accuracy", "name": "GPQA Diamond", "score": 92.3, "source": "https://openai.com/index/gpt-5-6/" }, { "dataset": "Tier 1-3", "date": "2026-07-09", "metric": "accuracy", "name": "FrontierMath", "score": 78.6, "source": "https://openai.com/index/gpt-5-6/", "version": "v2" }, { "date": "2026-07-09", "metric": "accuracy", "name": "BrowseComp", "score": 83.3, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "OSWorld", "score": 45.6, "source": "https://openai.com/index/gpt-5-6/", "version": "2.0" }, { "date": "2026-07-09", "metric": "accuracy", "name": "MMMU Pro", "score": 78.4, "source": "https://openai.com/index/gpt-5-6/", "variant": "no tools" }, { "date": "2026-07-09", "name": "Agents' Last Exam", "score": 50.3, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Toolathlon", "score": 53.4, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 51.2, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "4.1" }, { "date": "2026-07-09", "harness": "Codex", "metric": "index score", "name": "Artificial Analysis Coding Agent Index", "score": 74.6, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.1875, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 448 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 346.67, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 17.3, "source": "https://www.tbench.ai/?efforts=all&version=4.0", "variant": "95% CI ±2.8 percentage points", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 3.3, "source": "https://www.tbench.ai/news/terminal-bench-science-0-1", "variant": "3 trials per task", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "Max reasoning", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 31.4, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.2 percentage points" }, { "dataset": "100 tasks", "effort": "Default", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 21.4, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.2 percentage points" }, { "dataset": "May 2025 HLE revision; 2,158 text-only questions", "effort": "Xhigh", "harness": "GPT-5.6 Luna (medium) equality checker", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 37, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-sol-xhigh-vs-gpt-5-6-luna-xhigh", "variant": "pass@1" }, { "dataset": "May 2025 HLE revision; 2,158 text-only questions", "effort": "High", "harness": "GPT-5.6 Luna (medium) equality checker", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 33, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-sol-xhigh-vs-gpt-5-6-luna-high", "variant": "pass@1" }, { "dataset": "220 tasks across 44 occupations", "effort": "Non-reasoning", "harness": "Stirrup; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 928, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 0.3312633263840317, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 23.7902433286348, "source": "https://artificialanalysis.ai/models/gpt-5-6-luna", "version": "5.0.1" }, { "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 56.85840707964602, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 44.24778761061947, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 11.283185840707963, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 1.5486725663716814, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 0.731646, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 11.616, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Cost-efficient GPT-5.6 model for fast, high-volume workloads", "family": "gpt-luna", "knowledge": "2026-02-16", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.6 Luna", "open_weights": false, "release_date": "2026-07-09", "schema": 2 }, "openai/gpt-5.6-sol": { "benchmarks": [ { "date": "2026-07-09", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 64.6, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Terminal-Bench", "score": 88.8, "source": "https://openai.com/index/gpt-5-6/", "version": "2.1" }, { "date": "2026-07-09", "metric": "resolve rate", "name": "DeepSWE", "score": 72.7, "source": "https://openai.com/index/gpt-5-6/", "version": "1.1" }, { "date": "2026-07-09", "metric": "accuracy", "name": "GPQA Diamond", "score": 94.6, "source": "https://openai.com/index/gpt-5-6/" }, { "dataset": "Tier 1-3", "date": "2026-07-09", "metric": "accuracy", "name": "FrontierMath", "score": 89, "source": "https://openai.com/index/gpt-5-6/", "version": "v2" }, { "date": "2026-07-09", "metric": "accuracy", "name": "BrowseComp", "score": 90.4, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "OSWorld", "score": 62.6, "source": "https://openai.com/index/gpt-5-6/", "version": "2.0" }, { "date": "2026-07-09", "metric": "accuracy", "name": "MMMU Pro", "score": 83, "source": "https://openai.com/index/gpt-5-6/", "variant": "no tools" }, { "date": "2026-07-09", "name": "Agents' Last Exam", "score": 52.7, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Toolathlon", "score": 58, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 58.9, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "4.1" }, { "date": "2026-07-09", "harness": "Codex", "metric": "index score", "name": "Artificial Analysis Coding Agent Index", "score": 80, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 72.66666666666667, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 450 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "Full; 642 tasks", "effort": "Xhigh", "harness": "Codex", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 95.5, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "95% CI ±1.40 percentage points", "version": "2.0.0" }, { "cost": 2541.7, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 37.3, "source": "https://www.tbench.ai/?efforts=all&version=4.0", "variant": "95% CI ±3.8 percentage points", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 22.4, "source": "https://www.tbench.ai/news/terminal-bench-science-0-1", "variant": "3 trials per task", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "Max reasoning", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 45.0, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.1 percentage points" }, { "dataset": "100 tasks", "effort": "Default", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 39.5, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.2 percentage points" }, { "dataset": "May 2025 HLE revision; 2,158 text-only questions", "effort": "Max", "harness": "GPT-5.6 Luna (medium) equality checker", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 49, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-sol-xhigh-vs-gpt-5-6-sol", "variant": "pass@1" }, { "dataset": "220 tasks across 44 occupations", "effort": "Max", "harness": "Stirrup; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 1609, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-sol-xhigh-vs-gpt-5-6-sol", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations", "effort": "Non-reasoning", "harness": "Stirrup; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 1250, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 70.73170731707317, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 69.40133037694012, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 61.06194690265486, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 45.35398230088495, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 7.983597, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 37.879, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows", "family": "gpt-sol", "knowledge": "2026-02-16", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.6 Sol", "open_weights": false, "release_date": "2026-07-09", "schema": 2 }, "openai/gpt-5.6-terra": { "benchmarks": [ { "date": "2026-07-09", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 63.4, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Terminal-Bench", "score": 87.4, "source": "https://openai.com/index/gpt-5-6/", "version": "2.1" }, { "date": "2026-07-09", "metric": "resolve rate", "name": "DeepSWE", "score": 69.6, "source": "https://openai.com/index/gpt-5-6/", "version": "1.1" }, { "date": "2026-07-09", "metric": "accuracy", "name": "GPQA Diamond", "score": 92.9, "source": "https://openai.com/index/gpt-5-6/" }, { "dataset": "Tier 1-3", "date": "2026-07-09", "metric": "accuracy", "name": "FrontierMath", "score": 84.9, "source": "https://openai.com/index/gpt-5-6/", "version": "v2" }, { "date": "2026-07-09", "metric": "accuracy", "name": "BrowseComp", "score": 87.5, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "OSWorld", "score": 50.2, "source": "https://openai.com/index/gpt-5-6/", "version": "2.0" }, { "date": "2026-07-09", "metric": "accuracy", "name": "MMMU Pro", "score": 80.7, "source": "https://openai.com/index/gpt-5-6/", "variant": "no tools" }, { "date": "2026-07-09", "name": "Agents' Last Exam", "score": 50.4, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "success rate", "name": "Toolathlon", "score": 53.1, "source": "https://openai.com/index/gpt-5-6/" }, { "date": "2026-07-09", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 55, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "4.1" }, { "date": "2026-07-09", "harness": "Codex", "metric": "index score", "name": "Artificial Analysis Coding Agent Index", "score": 77.4, "source": "https://artificialanalysis.ai/articles/gpt-5-6-has-landed", "variant": "max", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "Pier; mini-swe-agent on Modal", "metric": "pass@1", "name": "DeepSWE", "score": 70, "source": "https://github.com/datacurve-ai/deep-swe/issues/82", "version": "1.1" }, { "dataset": "Full; 642 tasks", "effort": "Xhigh", "harness": "Codex", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 92.37, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "95% CI ±1.81 percentage points", "version": "2.0.0" }, { "cost": 1733.52, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "Max", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 21.5, "source": "https://www.tbench.ai/?efforts=all&version=4.0", "variant": "95% CI ±3.3 percentage points", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 8.6, "source": "https://www.tbench.ai/news/terminal-bench-science-0-1", "variant": "3 trials per task", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "Max reasoning", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 34.0, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.2 percentage points" }, { "dataset": "100 tasks", "effort": "Default", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 26.6, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.3 percentage points" }, { "dataset": "May 2025 HLE revision; 2,158 text-only questions", "effort": "Max", "harness": "GPT-5.6 Luna (medium) equality checker", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-astra-xhigh-vs-gpt-5-6-terra", "variant": "pass@1" }, { "dataset": "220 tasks across 44 occupations", "effort": "Non-reasoning", "harness": "Stirrup; shell and web browsing", "metric": "Elo rating", "name": "GDPval-AA", "score": 1105, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "cost": 3.734901899414753, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 32.3433563656461, "source": "https://artificialanalysis.ai/models/gpt-5-6-terra", "version": "5.0.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 69.62305986696231, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 60.17699115044248, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.76106194690266, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 35.11111111111111, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 450 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 24.053452115812917, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 5.602118, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 22.727, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Balanced GPT-5.6 model for capable, cost-efficient everyday work", "family": "gpt-terra", "knowledge": "2026-02-16", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-5.6 Terra", "open_weights": false, "release_date": "2026-07-09", "schema": 2 }, "openai/gpt-6-astra": { "benchmarks": [ { "name": "Agents' Last Exam", "score": 59.3, "source": "https://openai.com/index/gpt-6-astra/" }, { "dataset": "V2-Offline (v2026.08.08)", "metric": "partial score", "name": "OSWorld", "score": 72.6, "source": "https://openai.com/index/gpt-6-astra/", "version": "2.0" }, { "dataset": "108 real-world workflows", "effort": "Max", "harness": "Common internal framework", "metric": "binary task completion rate", "name": "OSWorld", "score": 34.3, "source": "https://dev.meta.ai/resources/blog/build-with-muse-wonder", "version": "osworld-v2.1" }, { "metric": "accuracy", "name": "ScreenSpot-Pro", "score": 92.7, "source": "https://openai.com/index/gpt-6-astra/", "variant": "no tools" }, { "metric": "success rate", "name": "AutomationBench", "score": 41.4, "source": "https://openai.com/index/gpt-6-astra/" }, { "metric": "geometric overlap", "name": "BenchCAD", "score": 95.9, "source": "https://openai.com/index/gpt-6-astra/", "variant": "with tools" }, { "metric": "accuracy", "name": "BrowseComp", "score": 91.5, "source": "https://openai.com/index/gpt-6-astra/" }, { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 57.9, "source": "https://openai.com/index/gpt-6-astra/", "version": "4.0.0" }, { "metric": "pass@1", "name": "DeepSWE", "score": 74.1, "source": "https://openai.com/index/gpt-6-astra/", "version": "1.1" }, { "dataset": "Extended", "metric": "score", "name": "FrontierCode", "score": 64.5, "source": "https://openai.com/index/gpt-6-astra/", "version": "1.1" }, { "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 64.6, "source": "https://openai.com/index/gpt-6-astra/", "version": "0.1.0" }, { "dataset": "Tier 4", "metric": "accuracy", "name": "FrontierMath", "score": 97.6, "source": "https://openai.com/index/gpt-6-astra/", "version": "v2" }, { "metric": "accuracy", "name": "GPQA Diamond", "score": 96.0, "source": "https://openai.com/index/gpt-6-astra/" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 57.2, "source": "https://openai.com/index/gpt-6-astra/", "variant": "with tools" }, { "metric": "success rate", "name": "ExploitBench", "score": 100.0, "source": "https://openai.com/index/gpt-6-astra/", "variant": "without production safeguards" }, { "metric": "success rate", "name": "SRE-Bench", "score": 88.0, "source": "https://openai.com/index/gpt-6-astra/", "variant": "single attempt" }, { "harness": "Responses API", "metric": "RHAE", "name": "ARC-AGI-3", "score": 99.9, "source": "https://openai.com/index/gpt-6-astra/" }, { "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 61, "source": "https://artificialanalysis.ai/articles/benchmarking-gpt-6-astra", "version": "4.1.1" }, { "effort": "Max", "harness": "Codex", "metric": "index score", "name": "Artificial Analysis Coding Agent Index", "score": 67, "source": "https://artificialanalysis.ai/articles/benchmarking-gpt-6-astra", "version": "1.4" }, { "dataset": "held-out private evaluation set", "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 41.4, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "XHigh", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 38.96, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 37.14, "source": "https://zapier.com/benchmarks", "version": "1.0.6" }, { "effort": "Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 71, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge", "version": "unspecified" }, { "cost": 7.497827996681416, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.23008849557522, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 452 scored attempts; errors excluded", "version": "1.1" }, { "cost": 4.429117203539823, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 74.11504424778761, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 452 scored attempts; errors excluded", "version": "1.1" }, { "cost": 3.9237237909292038, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 73.23008849557522, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 452 scored attempts; errors excluded", "version": "1.1" }, { "cost": 3.0754742975663714, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 72.78761061946902, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 452 scored attempts; errors excluded", "version": "1.1" }, { "cost": 1.595223439159292, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.03539823008849, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 452 scored attempts; errors excluded", "version": "1.1" }, { "dataset": "220 public GDPval tasks", "effort": "Max", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1583.69, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "220 public GDPval tasks", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1569.36, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "220 public GDPval tasks", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1536.04, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "220 public GDPval tasks", "effort": "Medium", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1498.47, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "220 public GDPval tasks", "effort": "Low", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1403.26, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "version": "2.1" }, { "dataset": "Full (642 tasks)", "effort": "High", "harness": "Codex", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 96.9, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "variant": "offline agent phase; fresh-sandbox re-grading", "version": "2.0.0" }, { "cost": 8.498119471851334, "cost_source": "https://artificialanalysis.ai/models/gpt-6-astra", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 59.0909090909091, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 500-step limit", "version": "4.0.0" }, { "cost": 5.855482520522051, "cost_source": "https://artificialanalysis.ai/models/gpt-6-astra-xhigh", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 59.5959595959596, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 500-step limit", "version": "4.0.0" }, { "cost": 4.0469926217499435, "cost_source": "https://artificialanalysis.ai/models/gpt-6-astra-high", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 54.040404040404, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 500-step limit", "version": "4.0.0" }, { "cost": 4.430107637760555, "cost_source": "https://artificialanalysis.ai/models/gpt-6-astra-medium", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 49.4949494949495, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 500-step limit", "version": "4.0.0" }, { "cost": 2.2485580260976215, "cost_source": "https://artificialanalysis.ai/models/gpt-6-astra-low", "cost_unit": "USD/task", "dataset": "Full (66 tasks)", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 41.9191919191919, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 500-step limit", "version": "4.0.0" }, { "dataset": "Full (70 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 63.3333333333333, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 1000-step limit", "version": "0.1.0" }, { "dataset": "Full (70 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 54.7619047619048, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 1000-step limit", "version": "0.1.0" }, { "dataset": "Full (70 tasks)", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 59.5238095238095, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 1000-step limit", "version": "0.1.0" }, { "dataset": "Full (70 tasks)", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 53.3333333333333, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 1000-step limit", "version": "0.1.0" }, { "dataset": "Full (70 tasks)", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 37.1428571428571, "source": "https://artificialanalysis.ai/models/releases/gpt-6-astra", "variant": "pass@1 averaged over 3 repeats; 1000-step limit", "version": "0.1.0" }, { "cost": 9.584259, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Full (66 tasks)", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 59.596, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Full (70 tasks)", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 62.86, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single pass@1 run; ungraded infrastructure/provider errors retried; 8-hour task limit", "version": "0.1.0" }, { "cost": 21.277828, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 65.023, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 3.6589264906126227, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 55.3057468569681, "source": "https://artificialanalysis.ai/models/gpt-6-astra-xhigh", "version": "5.0.1" }, { "cost": 6.244666943602518, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 56.8538559596797, "source": "https://artificialanalysis.ai/models/gpt-6-astra", "version": "5.0.1" }, { "cost": 1.4237229635663595, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 43.3349823228524, "source": "https://artificialanalysis.ai/models/gpt-6-astra-low", "version": "5.0.1" }, { "cost": 3.2385719685518306, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 54.849838322506, "source": "https://artificialanalysis.ai/models/gpt-6-astra-high", "version": "5.0.1" }, { "cost": 2.732438933013427, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 51.5548393864113, "source": "https://artificialanalysis.ai/models/gpt-6-astra-medium", "version": "5.0.1" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.8, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.7, "source": "https://artificialanalysis.ai/models/gpt-6-astra", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 63.8, "source": "https://artificialanalysis.ai/models/gpt-6-astra", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "GPT-6 Astra is OpenAI's most capable model for complex reasoning, coding, computer use, research, and document creation.", "family": "gpt-astra", "knowledge": "2026-04-30", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6 Astra", "open_weights": false, "release_date": "2026-09-04", "schema": 2 }, "openai/gpt-6-astra-fast": { "benchmarks": [ { "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 57, "source": "https://artificialanalysis.ai/providers/openai/", "variant": "Fast mode", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Fast variant of GPT-6 Astra for low-latency assistance and high-volume workloads.", "family": "gpt-astra", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6 Astra (Fast)", "open_weights": false, "release_date": "2026-09-04", "schema": 2 }, "openai/gpt-6-cyber": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Cybersecurity model for finding and validating exploitable vulnerabilities in authorized security research", "family": "gpt", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "GPT-6 Cyber", "open_weights": false, "release_date": "2026-11-05", "schema": 2 }, "openai/gpt-6-luna": { "benchmarks": [ { "dataset": "113 tasks", "effort": "Max", "metric": "score", "name": "DeepSWE", "score": 66.6, "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "version": "1.1" }, { "cost": 0.354726, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 13.636, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (three full runs; each run scored pass@1)", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": "4.29 ± 2.44", "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1; single run", "version": "0.1.0" }, { "effort": "Low", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 1.2, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "variant": "Zapier official leaderboard pass rate", "version": "1.0.6" }, { "effort": "Medium", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 9.4, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "variant": "Zapier official leaderboard pass rate", "version": "1.0.6" }, { "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 14.5, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "variant": "Zapier official leaderboard pass rate", "version": "1.0.6" }, { "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 20.7, "source": "https://zapier.com/benchmarks/gemini-carbon-4-report", "variant": "best observed published pass rate", "version": "1.0.6" }, { "dataset": "full 100-task set", "effort": "Max", "harness": "Inspect AI", "metric": "accuracy/mean", "name": "Chartography", "score": 29.1, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "cost": 0.15195575732105837, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 27.8552004938153, "source": "https://artificialanalysis.ai/models/gpt-6-luna/", "version": "5.0.1" }, { "cost": 0.10409320921120957, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 23.4508728089378, "source": "https://artificialanalysis.ai/models/gpt-6-luna-xhigh", "version": "5.0.1" }, { "cost": 0.07294776473194697, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 21.4290170864582, "source": "https://artificialanalysis.ai/models/gpt-6-luna-high", "version": "5.0.1" }, { "cost": 0.042201836106537785, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 18.4542156192416, "source": "https://artificialanalysis.ai/models/gpt-6-luna-medium", "version": "5.0.1" }, { "cost": 0.005115255188520391, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 9.66610812336462, "source": "https://artificialanalysis.ai/models/gpt-6-luna-low", "version": "5.0.1" }, { "cost": 0.07141741932294249, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 9.44244373619813, "source": "https://artificialanalysis.ai/models/gpt-6-luna-non-reasoning", "variant": "Non-reasoning", "version": "5.0.1" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": "1052 (-22/+22)", "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Non-reasoning", "version": "2.1" }, { "cost": 0.450147, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 52.322, "source": "https://www.vals.ai/models/openai_gpt-6-luna", "variant": "default provider: OpenAI; fallback rate: 0%", "version": "2.2" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 38.5, "source": "https://artificialanalysis.ai/models/gpt-6-luna", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 22.2, "source": "https://artificialanalysis.ai/models/gpt-6-luna", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "OpenAI's most efficient model for focused, high-volume tasks", "family": "gpt-luna", "knowledge": "2026-05-18", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6 Luna", "open_weights": false, "release_date": "2026-09-22", "schema": 2 }, "openai/gpt-6-sol": { "benchmarks": [ { "dataset": "113 tasks", "effort": "Max", "metric": "score", "name": "DeepSWE", "score": 68.8, "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "version": "1.1" }, { "dataset": "held-out private evaluation set", "effort": "Xhigh", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 33.2, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "API mode; up to 50 steps per task", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "Low", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 21.2, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "API mode; up to 50 steps per task", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "Medium", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 26.9, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "API mode; up to 50 steps per task", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "High", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 31.2, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "API mode; up to 50 steps per task", "version": "1.0.6" }, { "dataset": "held-out private evaluation set", "effort": "Max", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 32.0, "source": "https://zapier.com/benchmarks/claude-sonnet5_5-report", "variant": "API mode; up to 50 steps per task", "version": "1.0.6" }, { "effort": "Max", "metric": "score", "name": "Agents' Last Exam", "score": 56.4, "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "version": "1" }, { "dataset": "offline v2026.08.08", "effort": "Xhigh", "metric": "partial reward", "name": "OSWorld", "score": 60.5, "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna/", "version": "2.0" }, { "effort": "Max", "metric": "accuracy", "name": "Terminal-Bench", "score": 34.34, "source": "https://www.vals.ai/models/openai_gpt-6-sol", "version": "4.0.0" }, { "cost": 5.79322, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 44.444, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 across three runs; each run pass@1; reported ±3.54", "version": "4.0.0" }, { "effort": "Max", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 31.43, "source": "https://www.vals.ai/models/openai_gpt-6-sol" }, { "dataset": "70 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 30.0, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single-run pass@1; reported ±5.52", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "Max", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 53.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Source metric accuracy/mean; 10 runs per task; Gemini 3.5 Flash judge; Max reasoning" }, { "cost": 2.465800491412395, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 44.1555010304265, "source": "https://artificialanalysis.ai/models/gpt-6-sol", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "OpenAI model for complex coding and agentic workflows", "family": "gpt-sol", "knowledge": "2026-04-20", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6 Sol", "open_weights": false, "release_date": "2026-09-22", "schema": 2 }, "openai/gpt-6.1-sol": { "benchmarks": [ { "dataset": "V2 Full (642 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 96.11, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "version": "2.0.0" }, { "cost": 634.24, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "all 66 tasks", "effort": "Max", "harness": "Codex", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 58.2, "source": "https://www.tbench.ai/?efforts=all&version=4.0", "version": "4.0.0" }, { "cost": 1.72496, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "all 66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 55.051, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; three full runs, each run scored pass@1", "version": "4.0.0" }, { "cost": 1.8227634618325743, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-sol", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 56, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-vs-gpt-6-sol", "variant": "pass@1 averaged over three repeats; 500 agent step cap", "version": "4.0.0" }, { "cost": 1.0259491542428139, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-sol-xhigh", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 54, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-xhigh-vs-gpt-6-sol-low", "variant": "pass@1 averaged over three repeats; 500 agent step cap", "version": "4.0.0" }, { "cost": 0.8281767981765117, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-sol-high", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 52, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-high-vs-gpt-6-1-sol-low", "variant": "pass@1 averaged over three repeats; 500 agent step cap", "version": "4.0.0" }, { "cost": 0.6137554705459501, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-sol-medium", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 48, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-medium-vs-gpt-6-sol-medium", "variant": "pass@1 averaged over three repeats; 500 agent step cap", "version": "4.0.0" }, { "cost": 0.3829418451399688, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-sol-low", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 31, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-low-vs-gpt-6-1-sol-xhigh", "variant": "pass@1 averaged over three repeats; 500 agent step cap", "version": "4.0.0" }, { "dataset": "all 70 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 52.86, "source": "https://vals-ai.com/benchmarks/terminal-bench-science", "variant": "single run per model; pass@1", "version": "0.1.0" }, { "dataset": "all 70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 58, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-vs-gpt-6-sol", "variant": "pass@1 averaged over three repeats; 1,000 agent step cap", "version": "0.1.0" }, { "dataset": "all 70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 50, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-xhigh-vs-gpt-6-sol-low", "variant": "pass@1 averaged over three repeats; 1,000 agent step cap", "version": "0.1.0" }, { "dataset": "all 70 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 46, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-high-vs-gpt-6-1-sol-low", "variant": "pass@1 averaged over three repeats; 1,000 agent step cap", "version": "0.1.0" }, { "dataset": "all 70 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 37, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-medium-vs-gpt-6-sol-medium", "variant": "pass@1 averaged over three repeats; 1,000 agent step cap", "version": "0.1.0" }, { "dataset": "all 70 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 11, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-1-sol-low-vs-gpt-6-1-sol-xhigh", "variant": "pass@1 averaged over three repeats; 1,000 agent step cap", "version": "0.1.0" }, { "cost": 3.614054, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "finance, coding, legal, and tax tasks", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 63.944, "source": "https://www.vals.ai/models/openai_gpt-6.1-sol", "version": "2.2" }, { "cost": 0.7639949356156494, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 52.458219253291, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol-xhigh", "version": "5.0.1" }, { "cost": 0.5909853515780825, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 50.3291231936536, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol-high", "version": "5.0.1" }, { "cost": 0.3593898648025732, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 46.3468797067114, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol-medium", "version": "5.0.1" }, { "cost": 1.2720632679446995, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 54.6087338138148, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol", "version": "5.0.1" }, { "cost": 0.16147357831822573, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34.8029244314559, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol-low", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 52.9, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 58.8, "source": "https://artificialanalysis.ai/models/gpt-6-1-sol", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Near-Astra performance for complex coding, computer use, and professional work at a lower cost.", "family": "gpt-sol", "knowledge": "2026-04-30", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6.1 Sol", "open_weights": false, "release_date": "2026-09-29", "schema": 2 }, "openai/gpt-6.1-terra": { "benchmarks": [ { "effort": "Maximum reasoning effort", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 50.3, "source": "https://openai.com/index/introducing-gpt-6-1-terra/", "version": "4.0.0" }, { "cost": 1.4853125323157166, "cost_source": "https://artificialanalysis.ai/models/gpt-6-1-terra", "cost_unit": "USD/task", "dataset": "all 66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 55.6, "source": "https://artificialanalysis.ai/models/releases/comparisons/gpt-6-1-terra-vs-gpt-6-astra", "variant": "pass@1 averaged over three repeats", "version": "4.0.0" }, { "dataset": "all 70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 57.6, "source": "https://artificialanalysis.ai/models/releases/comparisons/gpt-6-1-terra-vs-gpt-6-astra", "variant": "pass@1 averaged over three repeats", "version": "0.1.0" }, { "dataset": "OpenAI GDPval (220 tasks)", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1286, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "Non-reasoning", "version": "2.1" }, { "cost": 0.22589585897456366, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 30.515898867357, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra-non-reasoning", "variant": "Non-reasoning", "version": "5.0.1" }, { "cost": 0.961108031081634, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 51.3265188577015, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra", "version": "5.0.1" }, { "cost": 0.16880458238389412, "cost_unit": "USD/weighted task", "effort": "Medium", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 38.1652076677975, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra-medium", "version": "5.0.1" }, { "cost": 0.07133756205169729, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 29.4169280792131, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra-low", "version": "5.0.1" }, { "cost": 0.40068900166662513, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 45.250051789066, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra-xhigh", "version": "5.0.1" }, { "cost": 0.2592796466169722, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 42.4344508712563, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra-high", "version": "5.0.1" }, { "cost": 2.939376, "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 60.536, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "version": "2.2" }, { "cost": 2.15663, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 49.495, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 46.2, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 47.3, "source": "https://artificialanalysis.ai/models/gpt-6-1-terra", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Fast, efficient model for visual reasoning, precise computer use, and clear, natural writing.", "family": "gpt-terra", "knowledge": "2026-06-01", "limit": { "context": 1050000, "input": 922000, "output": 128000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "GPT-6.1 Terra", "open_weights": false, "release_date": "2026-10-26", "schema": 2 }, "openai/gpt-audio": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Audio model for speech input and output with the Chat Completions API", "family": "gpt", "knowledge": "2023-10-01", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text", "audio/wav", "audio/aac", "audio/mpeg", "audio/flac", "audio/opus", "audio/pcm" ] }, "name": "GPT-Audio", "open_weights": false, "release_date": "2025-08-28", "schema": 2, "type": "chat", "v2_only": true }, "openai/gpt-audio-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Audio model for speech input and output with the Chat Completions API", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text", "audio/wav", "audio/aac", "audio/mpeg", "audio/flac", "audio/opus", "audio/pcm" ] }, "name": "GPT-Audio-1.5", "open_weights": false, "release_date": "2026-02-23", "schema": 2, "type": "chat", "v2_only": true }, "openai/gpt-audio-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Cost-efficient audio model for speech input and output with the Chat Completions API", "family": "gpt-mini", "knowledge": "2023-10-01", "limit": { "context": 128000, "output": 16384 }, "modalities": { "input": [ "text", "audio/wav", "audio/mpeg" ], "output": [ "text", "audio/wav", "audio/aac", "audio/mpeg", "audio/flac", "audio/opus", "audio/pcm" ] }, "name": "GPT-Audio Mini", "open_weights": false, "release_date": "2025-10-06", "schema": 2, "type": "chat", "v2_only": true }, "openai/gpt-image-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "OpenAI image model for production generation, edits, and brand-safe visual workflows", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT-Image-1", "open_weights": false, "release_date": "2025-04-24", "schema": 2, "type": "image" }, "openai/gpt-image-1-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Cost-efficient GPT Image 1 variant for image generation and editing", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT-Image-1 Mini", "open_weights": false, "release_date": "2025-10-06", "schema": 2, "type": "image", "v2_only": true }, "openai/gpt-image-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "text", "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT-Image-1.5", "open_weights": false, "release_date": "2025-11-25", "schema": 2, "type": "image" }, "openai/gpt-image-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT-Image-2", "open_weights": false, "release_date": "2026-04-21", "schema": 2, "type": "image" }, "openai/gpt-image-2.5-flare": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT Image 2.5 Flare", "open_weights": false, "release_date": "2026-09-08", "schema": 2, "type": "image" }, "openai/gpt-image-2.5-sunburst": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp" ] }, "name": "GPT Image 2.5 Sunburst", "open_weights": false, "release_date": "2026-09-08", "schema": 2, "type": "image" }, "openai/gpt-image-2.6-flare": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast image generation and editing with complexity control, markup-guided edits, SVG and animated GIF output", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/gif" ] }, "name": "GPT Image 2.6 Flare", "open_weights": false, "release_date": "2026-10-28", "schema": 2, "type": "image", "v2_only": true }, "openai/gpt-image-2.6-sunburst": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Detailed image generation and editing with complexity control, markup-guided edits, SVG and animated GIF output", "family": "gpt-image", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/gif" ] }, "name": "GPT Image 2.6 Sunburst", "open_weights": false, "release_date": "2026-10-28", "schema": 2, "type": "image", "v2_only": true }, "openai/gpt-live-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Full-duplex voice model for natural conversations that delegates reasoning and tools to a backend agent", "family": "gpt", "knowledge": "2025-07-31", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Live 1", "open_weights": false, "release_date": "2026-09-10", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-live-transcribe": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Low-latency streaming speech-to-text model with keyword and language hints", "family": "gpt", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "GPT-Live-Transcribe", "open_weights": false, "release_date": "2026-07-28", "schema": 2, "type": "transcription", "v2_only": true }, "openai/gpt-oss-120b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open GPT reasoning model for self-hosted agents and controllable deployments", "family": "gpt-oss", "knowledge": "2024-06", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 5.1, "sources": [ "https://huggingface.co/openai/gpt-oss-120b" ], "total_billion": 117 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT OSS 120B", "open_weights": true, "release_date": "2025-08-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openai/gpt-oss-120b" } ] }, "openai/gpt-oss-20b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open GPT reasoning model for self-hosted agents and controllable deployments", "family": "gpt-oss", "knowledge": "2024-06", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.6, "sources": [ "https://huggingface.co/openai/gpt-oss-20b" ], "total_billion": 21 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT OSS 20B", "open_weights": true, "release_date": "2025-08-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openai/gpt-oss-20b" } ] }, "openai/gpt-oss-safeguard-120b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "gpt-oss", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 5.1, "sources": [ "https://huggingface.co/openai/gpt-oss-safeguard-120b" ], "total_billion": 117 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT OSS Safeguard 120B", "open_weights": true, "release_date": "2025-10-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openai/gpt-oss-safeguard-120b" } ] }, "openai/gpt-oss-safeguard-20b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Safety model for policy screening, moderation, and risk-aware routing workflows", "family": "gpt-oss", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3.6, "sources": [ "https://huggingface.co/openai/gpt-oss-safeguard-20b" ], "total_billion": 21 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GPT OSS Safeguard 20B", "open_weights": true, "release_date": "2025-10-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b" } ] }, "openai/gpt-realtime": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Realtime speech-to-speech model for low-latency voice agents over WebRTC, WebSocket, or SIP", "family": "gpt", "knowledge": "2023-10-01", "limit": { "context": 32000, "output": 4096 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime", "open_weights": false, "release_date": "2025-08-28", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Realtime speech-to-speech model for voice agents and customer support", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 32000, "output": 4096 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime-1.5", "open_weights": false, "release_date": "2026-02-23", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-2": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Realtime speech-to-speech model with configurable reasoning and reliable tool use for voice agents", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 128000, "output": 32000 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime-2", "open_weights": false, "release_date": "2026-05-07", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-2.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Realtime speech-to-speech model with configurable reasoning, tool use, and robust voice-agent behavior", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 128000, "input": 96000, "output": 32000 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime-2.1", "open_weights": false, "release_date": "2026-07-06", "schema": 2, "type": "realtime" }, "openai/gpt-realtime-2.1-mini": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Distilled realtime reasoning model for faster, lower-cost voice interactions", "family": "gpt-mini", "knowledge": "2024-09-30", "limit": { "context": 128000, "output": 32000 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime-2.1 Mini", "open_weights": false, "release_date": "2026-07-06", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Cost-efficient realtime speech-to-speech model for voice agents", "family": "gpt-mini", "knowledge": "2023-10-01", "limit": { "context": 32000, "output": 4096 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma" ] }, "name": "GPT-Realtime Mini", "open_weights": false, "release_date": "2025-10-06", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-translate": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Streaming speech-to-speech translation model for live multilingual audio", "family": "gpt", "knowledge": "2024-09-30", "limit": { "context": 16000, "output": 2000 }, "modalities": { "input": [ "audio/pcm" ], "output": [ "text", "audio/pcm" ] }, "name": "GPT-Realtime-Translate", "open_weights": false, "release_date": "2026-05-07", "schema": 2, "type": "realtime", "v2_only": true }, "openai/gpt-realtime-whisper": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Streaming speech-to-text model for low-latency transcript deltas from live audio", "family": "whisper", "limit": { "context": 16000, "output": 2000 }, "modalities": { "input": [ "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "GPT Realtime Whisper", "open_weights": false, "release_date": "2026-05-07", "schema": 2, "type": "transcription" }, "openai/gpt-transcribe": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "High-accuracy speech-to-text model for audio files and committed Realtime turns", "family": "gpt", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm", "audio/pcm", "audio/pcmu", "audio/pcma" ], "output": [ "text" ] }, "name": "GPT-Transcribe", "open_weights": false, "release_date": "2026-07-28", "schema": 2, "type": "transcription", "v2_only": true }, "openai/o1": { "benchmarks": [ { "date": "2024-12-21", "metric": "percent correct", "name": "Aider Polyglot", "score": 61.7, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 7.96, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "O-series reasoning model for hard analysis, math, coding, and planning", "family": "o", "knowledge": "2023-09", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "o1", "open_weights": false, "release_date": "2024-12-05", "schema": 2 }, "openai/o1-pro": { "benchmarks": [ { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.12, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "O-series reasoning model for hard analysis, math, coding, and planning", "family": "o-pro", "knowledge": "2023-09", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "o1-pro", "open_weights": false, "release_date": "2025-03-19", "schema": 2 }, "openai/o3": { "benchmarks": [ { "date": "2025-06-25", "metric": "percent correct", "name": "Aider Polyglot", "score": 81.3, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "high", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 20.32, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "medium", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 19.2, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Deliberate o-series reasoner for hard math, coding, and multi-step analysis", "family": "o", "knowledge": "2024-05", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "application/pdf" ], "output": [ "text" ] }, "name": "o3", "open_weights": false, "release_date": "2025-04-16", "schema": 2 }, "openai/o3-deep-research": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Research model for long-horizon investigation, synthesis, and analytical reports", "family": "o", "knowledge": "2024-05", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "o3-deep-research", "open_weights": false, "release_date": "2025-06-26", "schema": 2 }, "openai/o3-mini": { "benchmarks": [ { "date": "2025-01-31", "metric": "percent correct", "name": "Aider Polyglot", "score": 60.4, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Smaller o-series reasoner for economical coding, math, and planning tasks", "family": "o-mini", "knowledge": "2024-05", "last_updated": "2025-01-29", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "o3-mini", "open_weights": false, "release_date": "2024-12-20", "schema": 2 }, "openai/o3-pro": { "benchmarks": [ { "date": "2025-06-28", "metric": "percent correct", "name": "Aider Polyglot", "score": 84.9, "source": "https://aider.chat/docs/leaderboards/" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "High-effort o3 tier for difficult technical reasoning and careful answers", "family": "o-pro", "knowledge": "2024-05", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "o3-pro", "open_weights": false, "release_date": "2025-06-10", "schema": 2 }, "openai/o4-mini": { "benchmarks": [ { "date": "2025-04-16", "metric": "percent correct", "name": "Aider Polyglot", "score": 72.0, "source": "https://aider.chat/docs/leaderboards/" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "high", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 18.08, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "effort": "medium", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 14.28, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Fast o-series model for compact reasoning, coding, and tool use", "family": "o-mini", "knowledge": "2024-05", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "o4-mini", "open_weights": false, "release_date": "2025-04-16", "schema": 2 }, "openai/o4-mini-deep-research": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Research model for long-horizon investigation, synthesis, and analytical reports", "family": "o-mini", "knowledge": "2024-05", "limit": { "context": 200000, "output": 100000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "o4-mini-deep-research", "open_weights": false, "release_date": "2025-06-26", "schema": 2 }, "openai/omni-moderation-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Moderation model that classifies potentially harmful content in text and images", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp" ], "output": [ "text" ] }, "name": "omni-moderation", "open_weights": false, "release_date": "2024-09-26", "schema": 2, "v2_only": true }, "openai/text-embedding-3-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Most capable OpenAI embedding model for English and non-English text", "family": "text-embedding", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "text-embedding-3-large", "open_weights": false, "release_date": "2024-01-25", "schema": 2, "type": "embedding", "v2_only": true }, "openai/text-embedding-3-small": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Small, efficient OpenAI text embedding model", "family": "text-embedding", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "text-embedding-3-small", "open_weights": false, "release_date": "2024-01-25", "schema": 2, "type": "embedding", "v2_only": true }, "openai/text-embedding-ada-002": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Legacy OpenAI text embedding model", "family": "text-embedding", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "text-embedding-ada-002", "open_weights": false, "release_date": "2022-12-15", "schema": 2, "type": "embedding", "v2_only": true }, "openai/tts-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-speech model optimized for low latency", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/opus", "audio/aac", "audio/flac", "audio/wav", "audio/pcm" ] }, "name": "TTS-1", "open_weights": false, "release_date": "2023-11-06", "schema": 2, "type": "speech", "v2_only": true }, "openai/tts-1-hd": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text-to-speech model optimized for audio quality", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/opus", "audio/aac", "audio/flac", "audio/wav", "audio/pcm" ] }, "name": "TTS-1 HD", "open_weights": false, "release_date": "2023-11-06", "schema": 2, "type": "speech", "v2_only": true }, "openai/whisper-1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose speech recognition model for transcription and translation", "family": "whisper", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm" ], "output": [ "text" ] }, "name": "Whisper", "open_weights": false, "release_date": "2023-03-01", "schema": 2, "type": "transcription", "v2_only": true }, "openai/whisper-large-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open Whisper checkpoint for robust multilingual transcription and captioning", "family": "whisper", "limit": { "context": 448, "output": 448 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 count for the complete checkpoint; the separately hosted FP32 format is not summed.", "sources": [ "https://huggingface.co/api/models/openai/whisper-large-v3" ], "total_billion": 1.54349056 } } }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm" ], "output": [ "text" ] }, "name": "Whisper 3 Large", "open_weights": true, "release_date": "2023-11-06", "schema": 2, "type": "transcription" }, "openai/whisper-large-v3-turbo": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech transcription model for accurate audio-to-text and captioning workflows", "family": "whisper", "limit": { "context": 448, "output": 448 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official model table reports 809M for turbo and identifies it as the optimized large-v3 model. Rounded published count retained.", "sources": [ "https://github.com/openai/whisper/blob/main/README.md" ], "total_billion": 0.809 } } }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm" ], "output": [ "text" ] }, "name": "Whisper Large v3 Turbo", "open_weights": true, "release_date": "2024-10-01", "schema": 2, "type": "transcription" }, "openai/whisper-tiny": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Smallest multilingual Whisper checkpoint for fast, low-cost speech transcription", "family": "whisper", "license": "Apache-2.0", "limit": { "context": 448, "output": 448 }, "modalities": { "input": [ "audio/flac", "audio/mpeg", "audio/mp4", "audio/ogg", "audio/wav", "audio/webm" ], "output": [ "text" ] }, "name": "Whisper Tiny", "open_weights": true, "release_date": "2022-09-21", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openai/whisper-tiny" } ] }, "openbmb/minicpm-v-4.7-1b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Compact MiniCPM-V model for single-image, multi-image and video understanding", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not just the language backbone.", "sources": [ "https://huggingface.co/api/models/openbmb/MiniCPM-V-4.7-1B" ], "total_billion": 1.554476272 } } }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "text" ] }, "name": "MiniCPM-V 4.7 1B", "open_weights": true, "release_date": "2026-10-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-1B" } ] }, "openbmb/minicpm-v-4.7-35b-a3b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "First MoE MiniCPM-V model for single-image, multi-image and video understanding", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint. Active count not inferred from the name.", "sources": [ "https://huggingface.co/api/models/openbmb/MiniCPM-V-4.7-35B-A3B" ], "total_billion": 35.212875824 } } }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "text" ] }, "name": "MiniCPM-V 4.7 35B-A3B", "open_weights": true, "release_date": "2026-10-07", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-35B-A3B" } ] }, "openbmb/minicpm5-1b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Dense 1B-class open-source model for on-device and resource-constrained use, with native long-context support, Think / No Think chat modes, and tool calling", "license": "apache-2.0", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Exact count explicitly reported in the card, including embeddings, converted to billions.", "sources": [ "https://huggingface.co/openbmb/MiniCPM5-1B" ], "total_billion": 1.080632832 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniCPM5-1B", "open_weights": true, "release_date": "2026-05-19", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openbmb/MiniCPM5-1B" } ] }, "openbmb/minicpm5-2b": { "benchmarks": [ { "dataset": "2,158 text-only questions from the 2,500-question HLE dataset", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.9, "source": "https://huggingface.co/openbmb/MiniCPM5-2B", "variant": "pass@1; GPT-5.6 Luna (medium) equality checker" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Dense 2B-class open-source model for on-device and resource-constrained use, with native long-context support, tool calling, and agentic tasks", "last_updated": "2026-09-12", "license": "apache-2.0", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Exact count explicitly reported in the card, including embeddings, converted to billions.", "sources": [ "https://huggingface.co/openbmb/MiniCPM5-2B" ], "total_billion": 2.51675648 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiniCPM5-2B", "open_weights": true, "release_date": "2026-09-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/openbmb/MiniCPM5-2B" } ] }, "opendatalab/mineru2.5-2509-1.2b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "1.2B-parameter vision-language model from OpenDataLab for two-step document parsing: global layout analysis on downsampled images, then fine-grained text, formula, and table recognition at native resolution", "license": "AGPL-3.0", "limit": { "context": 16384 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2509.22186" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "text" ] }, "name": "MinerU2.5 2509 1.2B", "open_weights": true, "release_date": "2025-09-17", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/opendatalab/MinerU2.5-2509-1.2B" } ] }, "paddlepaddle/paddleocr-vl": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Compact vision-language model for multilingual document parsing, OCR, tables, formulas, and charts", "license": "Apache-2.0", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official full vision-language model size, not the 0.3B language component.", "sources": [ "https://huggingface.co/PaddlePaddle/PaddleOCR-VL" ], "total_billion": 0.9 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "PaddleOCR VL", "open_weights": true, "release_date": "2025-10-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/PaddlePaddle/PaddleOCR-VL" } ] }, "perceptron/perceptron-mk1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multimodal reasoning model for visual analysis, planning, and tool use", "limit": { "context": 32768, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "video/mp4", "video/webm" ], "output": [ "text" ] }, "name": "Perceptron Mk1", "open_weights": false, "release_date": "2026-05-12", "schema": 2 }, "perceptron/perceptron-mk1.5": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Image, video, and audio understanding with visual grounding, reasoning, and function calling", "limit": { "context": 36864, "output": 8192 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "video/mp4", "video/quicktime", "video/webm", "audio/wav", "audio/mpeg", "audio/flac" ], "output": [ "text" ] }, "name": "Perceptron Mk1.5", "open_weights": false, "release_date": "2026-09-25", "schema": 2 }, "perplexity/pplx-decider-flash-v1.2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Smaller Perplexity decision model distilled onto Qwen3.5-4B for high-volume yes/no, choice and score questions about text or images", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Main checkpoint (without readout)", "notes": "Hub BF16 count independently verified against both main shard headers.", "sources": [ "https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2/blob/9ff7cf9838e59df461c276b7699d56b244d2af58/model.safetensors.index.json", "https://huggingface.co/api/models/perplexity-ai/pplx-decider-flash-v1.2" ], "total_billion": 4.539265536 }, { "name": "Readout", "notes": "Stored BF16 weight count verified from this separate head's tensor header.", "sources": [ "https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2/blob/9ff7cf9838e59df461c276b7699d56b244d2af58/readout.safetensors" ], "total_billion": 0.0006528 } ], "notes": "Main checkpoint and required readout are scoped separately; no complete total is inferred by adding components." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "pplx-decider-flash-v1.2", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2" } ] }, "perplexity/pplx-decider-v1.2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Perplexity's open-weight decision model on Qwen3.8-27B that returns probabilities for typed yes/no, choice and score questions about text or images", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Main checkpoint (without readout)", "notes": "Hub BF16 count independently verified against all eleven main shard headers.", "sources": [ "https://huggingface.co/perplexity-ai/pplx-decider-v1.2/blob/cce0f3f80033e9d89d80831d5281fd72124aacce/model.safetensors.index.json", "https://huggingface.co/api/models/perplexity-ai/pplx-decider-v1.2" ], "total_billion": 26.08533016 }, { "name": "Readout", "notes": "Stored BF16 weight count verified from this separate head's tensor header.", "sources": [ "https://huggingface.co/perplexity-ai/pplx-decider-v1.2/blob/cce0f3f80033e9d89d80831d5281fd72124aacce/readout.safetensors" ], "total_billion": 0.0013056 } ], "notes": "Main checkpoint and required readout are scoped separately; no complete total is inferred by adding components." } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "decision" ] }, "name": "pplx-decider-v1.2", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-decider-v1.2" } ] }, "perplexity/pplx-embed-context-v1-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open contextual text embedding model for document chunks that share surrounding context", "license": "MIT", "limit": { "context": 32000, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for this complete contextual embedding checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/perplexity-ai/pplx-embed-context-v1-0.6b" ], "total_billion": 0.59604992 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "pplx-embed-context-v1-0.6b", "open_weights": true, "release_date": "2026-02-26", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-embed-context-v1-0.6b" } ] }, "perplexity/pplx-embed-context-v1-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open contextual text embedding model for document chunks that share surrounding context", "license": "MIT", "limit": { "context": 32000, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for this complete contextual embedding checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/perplexity-ai/pplx-embed-context-v1-4b" ], "total_billion": 4.022468096 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "pplx-embed-context-v1-4b", "open_weights": true, "release_date": "2026-02-26", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-embed-context-v1-4b" } ] }, "perplexity/pplx-embed-v1-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open text embedding model for web-scale dense retrieval", "license": "MIT", "limit": { "context": 32000, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for this complete embedding checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/perplexity-ai/pplx-embed-v1-0.6b" ], "total_billion": 0.59604992 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "pplx-embed-v1-0.6b", "open_weights": true, "release_date": "2026-02-26", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-embed-v1-0.6b" } ] }, "perplexity/pplx-embed-v1-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open text embedding model for web-scale dense retrieval", "license": "MIT", "limit": { "context": 32000, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for this complete embedding checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/perplexity-ai/pplx-embed-v1-4b" ], "total_billion": 4.022468096 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "pplx-embed-v1-4b", "open_weights": true, "release_date": "2026-02-26", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/perplexity-ai/pplx-embed-v1-4b" } ] }, "perplexity/sonar": { "benchmarks": [ { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 22.9, "source": "https://openrouter.ai/perplexity/sonar/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Fast web-grounded Sonar for current answers, citations, and lightweight retrieval", "family": "sonar", "knowledge": "2025-09-01", "last_updated": "2025-09-01", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Sonar", "open_weights": false, "release_date": "2024-01-01", "schema": 2 }, "perplexity/sonar-deep-research": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Sonar search model for autonomous research and citation-backed long-form reports", "family": "sonar", "knowledge": "2025-01", "last_updated": "2025-09-01", "limit": { "context": 128000, "output": 32768 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Sonar Deep Research", "open_weights": false, "release_date": "2025-02-01", "schema": 2 }, "perplexity/sonar-pro": { "benchmarks": [ { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 22.6, "source": "https://openrouter.ai/perplexity/sonar-pro/benchmarks" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Deeper Sonar search model with broader retrieval and stronger synthesis", "family": "sonar-pro", "knowledge": "2025-09-01", "last_updated": "2025-09-01", "limit": { "context": 200000, "output": 8192 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Sonar Pro", "open_weights": false, "release_date": "2024-01-01", "schema": 2 }, "perplexity/sonar-reasoning-pro": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": true }, "description": "Web-grounded Sonar for multi-step research questions that need cited reasoning", "family": "sonar-reasoning", "knowledge": "2025-09-01", "last_updated": "2025-09-01", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Sonar Reasoning Pro", "open_weights": false, "release_date": "2024-01-01", "schema": 2 }, "pfnet/plamo-embedding-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Japanese text embedding model from Preferred Networks producing 2048-dimensional vectors", "family": "plamo", "license": "Apache-2.0", "limit": { "context": 4096, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "PLaMo Embedding 1B", "open_weights": true, "release_date": "2025-04-11", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/pfnet/plamo-embedding-1b" } ] }, "phocinae/phocinae-largha-150m-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Encoder decision model that answers typed questions (noul, choice, score) over text with calibrated probabilities, on CPU and GPU", "last_updated": "2026-10-09", "license": "Apache-2.0", "limit": { "context": 8192, "output": 0 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/Phocinae/Phocinae-Largha-150M-v1" } ], "metadata": { "modelatlas": { "parameters": { "notes": "The author explicitly reports 144,292,870 parameters in the model card, rounded there to 144.3M.", "sources": [ "https://huggingface.co/Phocinae/Phocinae-Largha-150M-v1" ], "total_billion": 0.14429287 } } }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Phocinae Largha 150M v1", "open_weights": true, "release_date": "2026-10-08", "schema": 2, "type": "decision", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Phocinae/Phocinae-Largha-150M-v1" } ] }, "pixilabai/blink-v0.3-26b-a4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "PixilabAI's fast 26B-A4B open-weight decision model on Gemma 4 that returns calibrated probabilities for typed choice, yes/no and score questions about text and images", "license": "Apache-2.0", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text", "image" ], "output": [ "decision" ] }, "name": "Blink v0.3 26B-A4B", "open_weights": true, "release_date": "2026-10-02", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face (NVFP4)", "url": "https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-NVFP4" }, { "label": "Hugging Face (FP8)", "url": "https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-FP8" } ] }, "poolside/laguna-m.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Poolside's open-weight model for agentic coding and long-horizon work", "family": "laguna", "last_updated": "2026-06-13", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 23, "sources": [ "https://huggingface.co/poolside/Laguna-M.1" ], "total_billion": 225 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Laguna M.1", "open_weights": true, "release_date": "2026-04-28", "schema": 2 }, "poolside/laguna-s-2.1": { "benchmarks": [ { "date": "2026-07-21", "effort": "Max thinking", "harness": "pool", "metric": "pass@1", "name": "DeepSWE", "score": 40.4, "source": "https://poolside.ai/blog/introducing-laguna-s-2-1", "variant": "pass@1 averaged over 3 attempts per task", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "118B Mixture-of-Experts agentic coding model from Poolside", "family": "laguna", "limit": { "context": 1048576, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 8, "notes": "Per-token active count is approximate in the official specification.", "sources": [ "https://huggingface.co/poolside/Laguna-S-2.1" ], "total_billion": 118 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Laguna S 2.1", "open_weights": true, "release_date": "2026-07-21", "schema": 2 }, "poolside/laguna-xs-2.1": { "benchmarks": [ { "date": "2026-07-02", "harness": "Harbor", "metric": "resolved", "name": "SWE-Bench Verified", "score": 70.9, "source": "https://poolside.ai/blog/introducing-laguna-xs-2-1" }, { "date": "2026-07-02", "harness": "Harbor", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 63.1, "source": "https://poolside.ai/blog/introducing-laguna-xs-2-1" }, { "date": "2026-07-02", "harness": "Harbor", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 47.6, "source": "https://poolside.ai/blog/introducing-laguna-xs-2-1" }, { "date": "2026-07-02", "harness": "Harbor", "metric": "success rate", "name": "Terminal-Bench", "score": 37.5, "source": "https://poolside.ai/blog/introducing-laguna-xs-2-1", "version": "2.0" }, { "dataset": "113 tasks", "metric": "pass@1", "name": "DeepSWE", "score": 0.3, "source": "https://poolside.ai/blog/introducing-laguna-s-2-1", "variant": "pass@1 averaged over 3 attempts per task", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Agentic coding model from Poolside in the XS size class for local deployment", "family": "laguna", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/poolside/Laguna-XS-2.1" ], "total_billion": 33 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Laguna XS 2.1", "open_weights": true, "release_date": "2026-07-02", "schema": 2 }, "poolside/laguna-xs.2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Agentic coding model from Poolside in the XS size class for local deployment", "family": "laguna", "last_updated": "2026-06-13", "limit": { "context": 262144, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 3, "sources": [ "https://huggingface.co/poolside/Laguna-XS.2" ], "total_billion": 33 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Laguna XS.2", "open_weights": true, "release_date": "2026-04-28", "schema": 2 }, "quiverai/arrow-2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Fast SVG generation model for creation, vectorization, editing, and animation", "limit": { "context": 131072, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "image/svg+xml" ], "output": [ "text", "image/svg+xml" ] }, "name": "Arrow 2", "open_weights": false, "release_date": "2026-09-16", "schema": 2 }, "quiverai/arrow-2-telos": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "High-fidelity SVG generation model for complex vector work and long-context refinement", "limit": { "context": 1050000, "output": 65536 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "image/svg+xml" ], "output": [ "text", "image/svg+xml" ] }, "name": "Arrow 2 Telos", "open_weights": false, "release_date": "2026-09-16", "schema": 2 }, "recraft/recraft-v3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for photorealism and text rendering", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V3", "open_weights": false, "release_date": "2024-10-30", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model with design taste and prompt accuracy at 1K", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4", "open_weights": false, "release_date": "2026-02-17", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "2K Recraft V4 image model with finer detail", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4 Pro", "open_weights": false, "release_date": "2026-02-17", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-pro-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-detail Recraft V4 model that generates editable SVG images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4 Pro Vector", "open_weights": false, "release_date": "2026-02-17", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-styles": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Style-consistent image model driven by one to ten reference images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4 Styles", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-styles-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "2K style-consistent image model driven by reference images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4 Styles Pro", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-styles-pro-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-detail style-consistent SVG image model driven by reference images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4 Styles Pro Vector", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-styles-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Style-consistent SVG image model driven by reference images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4 Styles Vector", "open_weights": false, "release_date": "2026-08-26", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Recraft V4 model that generates editable SVG images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4 Vector", "open_weights": false, "release_date": "2026-02-17", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Default Recraft image model for photorealism and illustration at 1K", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4.1", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast, low-cost text-to-image model with quality close to Recraft V4.1", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4.1 Flash", "open_weights": false, "release_date": "2026-09-23", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "2K Recraft V4.1 image model for print-ready detail", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4.1 Pro", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-pro-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-detail Recraft V4.1 model that generates editable SVG images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4.1 Pro Vector", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-utility": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Recraft V4.1 model with flat lighting and simple composition for mockups", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4.1 Utility", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-utility-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "2K Recraft V4.1 Utility model for product shots and mockups", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png" ] }, "name": "Recraft V4.1 Utility Pro", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-utility-pro-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-detail Recraft V4.1 Utility model that generates editable SVG images for mockups", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4.1 Utility Pro Vector", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-utility-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Recraft V4.1 Utility model that generates editable SVG images for mockups", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4.1 Utility Vector", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recraft/recraft-v4.1-vector": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Recraft V4.1 model that generates editable SVG images", "family": "recraft", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/svg+xml" ] }, "name": "Recraft V4.1 Vector", "open_weights": false, "release_date": "2026-05-14", "schema": 2, "type": "image", "v2_only": true }, "recursive/flint": { "benchmarks": [ { "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 89.7, "source": "https://huggingface.co/RecursiveSI/flint", "variant": "Average pass@1 over at least 5 attempts; no internet access; timeout up to 5 hours; temperature 1.0; top-p 0.95; top-k 20", "version": "2.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open-weight mixture-of-experts model with about 2.5B active parameters for local coding agents", "family": "flint", "knowledge": "2026-06", "license": "Apache-2.0", "limit": { "context": 262144, "output": 262144 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/RecursiveSI/flint" }, { "label": "Introducing Recursive Flint", "type": "announcement", "url": "https://www.recursive.com/updates/introducing-recursive-flint" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 2.5, "notes": "Approximately 2.5B active including the output head and excluding the full input embedding matrix; active body alone is approximately 2.2B.", "sources": [ "https://huggingface.co/RecursiveSI/flint" ], "total_billion": 39.8 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Flint", "open_weights": true, "release_date": "2026-10-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/RecursiveSI/flint" }, { "label": "Hugging Face (INT4)", "quantization": "int4", "url": "https://huggingface.co/RecursiveSI/flint-int4" } ] }, "reflection/beam-1.0-501b-a23b": { "benchmarks": [ { "dataset": "datacurve/deep-swe", "date": "2026-10-07", "metric": "pass@1", "name": "DeepSWE", "score": 43.3, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/Beam-1.0-501B-A23B.yaml", "variant": "BF16", "version": "1.1" }, { "dataset": "hard", "name": "SWE-Bench Pro", "score": 76.5, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16", "version": "2" }, { "dataset": "ScaleAI/SWE-bench_Pro", "date": "2026-10-07", "name": "SWE-Bench Pro", "score": 66.1, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/Beam-1.0-501B-A23B.yaml", "variant": "BF16", "version": "1" }, { "dataset": "harborframework/terminal-bench-2.1", "date": "2026-10-07", "name": "Terminal-Bench", "score": 80.1, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/Beam-1.0-501B-A23B.yaml", "variant": "BF16", "version": "2.1" }, { "date": "2026-10-16", "name": "SWE-Atlas Codebase QnA", "score": 31.8, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "SWE-Bench Multilingual", "score": 78.3, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "SWE-Bench Verified", "score": 81.6, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "AIME 2026", "score": 97.7, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "dataset": "cais/hle", "date": "2026-10-07", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 36.0, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/Beam-1.0-501B-A23B.yaml", "variant": "BF16; no tools" }, { "dataset": "cais/hle", "date": "2026-10-07", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 51.1, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/Beam-1.0-501B-A23B.yaml", "variant": "BF16; with tools" }, { "date": "2026-10-16", "name": "SciCode", "score": 49.2, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "CritPt", "score": 16.3, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "GPQA Diamond", "score": 90.6, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "LiveCodeBench", "score": 88.3, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "dataset": "public", "name": "AutomationBench", "score": 39.3, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "MCP Atlas", "score": 80.8, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "τ3 Banking", "score": 36.1, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "BrowseComp", "score": 76.8, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16; with context management" }, { "date": "2026-10-16", "name": "DeepSearchQA", "score": 80.5, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16; with context management" }, { "date": "2026-10-16", "name": "AA-LCR", "score": 78.2, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "date": "2026-10-16", "name": "LongBench", "score": 67.5, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16", "version": "2" }, { "date": "2026-10-16", "name": "IFBench", "score": 79.6, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "dataset": "public split", "date": "2026-10-16", "metric": "index", "name": "AA-Omniscience", "score": 10.8, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B", "variant": "BF16" }, { "dataset": "harborframework/terminal-bench", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 3.0, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/terminal-bench-4.0.yaml", "variant": "pass@1 averaged over 3 repeats", "version": "4.0.0" }, { "dataset": "harborframework/terminal-bench-science", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0.5, "source": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/.eval_results/terminal-bench-science.yaml", "variant": "pass@1 averaged over 3 repeats", "version": "0.1.0" }, { "metric": "performance score", "name": "Vals Index", "score": 43.42, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 0.28440671352465285, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 17.7307282547205, "source": "https://artificialanalysis.ai/models/reflection-beam", "version": "5.0.1" }, { "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 4.04, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34.7, "source": "https://artificialanalysis.ai/models/reflection-beam", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 12.8, "source": "https://artificialanalysis.ai/models/reflection-beam", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Sparse mixture-of-experts reasoning model with 23B active parameters for coding and agentic workflows", "family": "beam", "knowledge": "2026-06-30", "last_updated": "2026-10-16", "license": "Apache-2.0", "limit": { "context": 1048576, "output": 131072 }, "links": [ { "label": "Model card", "type": "model_card", "url": "https://huggingface.co/reflection/Beam-1.0-501B-A23B" }, { "label": "Launch announcement", "type": "announcement", "url": "https://reflection.ai/blog/beam-is-available" }, { "label": "Introducing Beam", "type": "blog", "url": "https://reflection.ai/blog/introducing-beam" }, { "label": "API docs", "type": "docs", "url": "https://developers.reflection.ai/models" } ], "metadata": { "modelatlas": { "parameters": { "active_billion": 23.4, "sources": [ "https://huggingface.co/reflection/Beam-1.0-501B-A23B" ], "total_billion": 501.3 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Beam 1.0 501B A23B", "open_weights": true, "release_date": "2026-10-05", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/reflection/Beam-1.0-501B-A23B" }, { "label": "Hugging Face (FP8)", "quantization": "fp8", "url": "https://huggingface.co/reflection/Beam-1.0-501B-A23B-FP8" }, { "label": "Hugging Face (NVFP4)", "quantization": "nvfp4", "url": "https://huggingface.co/reflection/Beam-1.0-501B-A23B-NVFP4" } ] }, "rekaai/reka-edge": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal model for analyzing text, images, documents, and rich media", "family": "reka", "limit": { "context": 16384, "output": 14745 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/RekaAI/reka-edge-2603" ], "total_billion": 7 } } }, "modalities": { "input": [ "image", "text", "video" ], "output": [ "text" ] }, "name": "Reka Edge", "open_weights": true, "release_date": "2026-03-11", "schema": 2 }, "rekaai/reka-flash-3": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Efficient model for low-latency assistance, extraction, and routine automation", "family": "reka", "knowledge": "2025-01-31", "limit": { "context": 65536, "output": 58982 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/RekaAI/reka-flash-3" ], "total_billion": 21 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Reka Flash 3", "open_weights": true, "release_date": "2025-03-10", "schema": 2 }, "relace/relace-apply-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose chat model for instruction following, writing, and analysis", "limit": { "context": 256000, "output": 128000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Relace Apply 3", "open_weights": false, "release_date": "2025-09-26", "schema": 2 }, "relace/relace-search": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Tool-capable chat model for instruction following and agentic application workflows", "limit": { "context": 256000, "output": 128000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Relace Search", "open_weights": false, "release_date": "2025-12-08", "schema": 2 }, "resemble-ai/chatterbox-nano": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Compact English text-to-speech model with zero-shot voice cloning", "license": "MIT", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "audio" ] }, "name": "Chatterbox Nano", "open_weights": true, "release_date": "2026-04-14", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ResembleAI/chatterbox-nano" } ] }, "respan/span-01": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Behavior classifier for agent traces that applies plain-language definitions and returns present, absent, and not-observable probabilities in one forward pass", "limit": { "context": 0, "output": 0 }, "links": [ { "type": "docs", "url": "https://www.respan.ai/docs/documentation/span-01/concept" }, { "type": "announcement", "url": "https://www.respan.ai/blog/introducing-span-1" } ], "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Span-01", "open_weights": false, "release_date": "2026-09-24", "schema": 2, "type": "decision" }, "respan/span-02": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight decision model with optional anytime reasoning over text, JSON state, and images, returning typed choices, yes/no probabilities, scores, and numbers", "license": "Apache-2.0", "limit": { "context": 131072, "output": 32 }, "links": [ { "type": "docs", "url": "https://www.respan.ai/docs/documentation/span-02/concept" }, { "type": "announcement", "url": "https://www.respan.ai/blog/introducing-span-2" }, { "type": "model_card", "url": "https://huggingface.co/respanai/span-02" } ], "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "text" ] }, "name": "Span-02", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "type": "decision", "weights": [ { "format": "safetensors", "label": "NVFP4/FP8 weights and Rust/CUDA inference engine", "quantization": "NVFP4/FP8", "url": "https://huggingface.co/respanai/span-02" } ] }, "runway/aleph-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "In-context video editing model that applies text and keyframe edits across existing footage", "family": "runway", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm", "video/3gpp", "video/ogg" ], "output": [ "video/mp4" ] }, "name": "Aleph 2.0", "open_weights": false, "release_date": "2026-06-02", "schema": 2, "type": "video", "v2_only": true }, "runway/gen-4.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Text- and image-to-video model for cinematic motion and prompt adherence", "family": "runway", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Gen-4.5", "open_weights": false, "release_date": "2025-12-01", "schema": 2, "type": "video", "v2_only": true }, "runwayml/aleph-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video editing model that changes existing footage from a text prompt and optional keyframe images", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska", "video/webm" ], "output": [ "video/mp4" ] }, "name": "Aleph 2.0", "open_weights": false, "release_date": "2026-06-02", "schema": 2, "type": "video", "v2_only": true }, "runwayml/gen-4.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video generation model for text-to-video and image-to-video with precise prompt adherence and physical accuracy", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Gen-4.5", "open_weights": false, "release_date": "2025-12-01", "schema": 2, "type": "video", "v2_only": true }, "sakana/fugu": { "benchmarks": [ { "name": "SWE Bench Pro", "score": 59.0, "source": "https://console.sakana.ai/models" }, { "name": "Terminal Bench 2.1", "score": 80.2, "source": "https://console.sakana.ai/models" }, { "name": "LiveCodeBench", "score": 92.9, "source": "https://console.sakana.ai/models" }, { "name": "LiveCodeBench Pro", "score": 87.8, "source": "https://console.sakana.ai/models" }, { "dataset": "Full 2,500-sample HLE, including multimodal samples", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 47.2, "source": "https://arxiv.org/html/2606.21228v2", "variant": "without tools", "version": "unspecified" }, { "name": "CharXiv Reasoning", "score": 85.1, "source": "https://console.sakana.ai/models" }, { "name": "GPQA Diamond", "score": 95.5, "source": "https://console.sakana.ai/models" }, { "name": "SciCode", "score": 60.1, "source": "https://console.sakana.ai/models" }, { "name": "τ3 Banking", "score": 21.7, "source": "https://console.sakana.ai/models" }, { "name": "Long Context Reasoning", "score": 74.7, "source": "https://console.sakana.ai/models" }, { "name": "MRCRv2", "score": 86.6, "source": "https://console.sakana.ai/models" }, { "name": "CTI-REALM", "score": 67.5, "source": "https://console.sakana.ai/models" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Multi-agent model for routing expert agents across complex analytical tasks", "family": "fugu", "limit": { "context": 1000000 }, "links": [ { "label": "Official model catalog", "type": "docs", "url": "https://raw.githubusercontent.com/SakanaAI/fugu/refs/heads/main/configs/files/fugu.json" } ], "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Fugu", "open_weights": false, "release_date": "2026-06-15", "schema": 2 }, "sakana/fugu-max": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Multi-agent model for routing expert agents across complex analytical tasks", "family": "fugu", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image", "application/pdf" ], "output": [ "text" ] }, "name": "Fugu Max", "open_weights": false, "release_date": "2026-09-11", "schema": 2 }, "sakana/fugu-ultra": { "benchmarks": [ { "name": "SWE Bench Pro", "score": 73.7, "source": "https://console.sakana.ai/models" }, { "name": "Terminal Bench 2.1", "score": 82.1, "source": "https://console.sakana.ai/models" }, { "name": "LiveCodeBench", "score": 93.2, "source": "https://console.sakana.ai/models" }, { "name": "LiveCodeBench Pro", "score": 90.8, "source": "https://console.sakana.ai/models" }, { "name": "Humanity’s Last Exam", "score": 50.0, "source": "https://console.sakana.ai/models" }, { "name": "CharXiv Reasoning", "score": 86.6, "source": "https://console.sakana.ai/models" }, { "name": "GPQA Diamond", "score": 95.5, "source": "https://console.sakana.ai/models" }, { "name": "SciCode", "score": 58.7, "source": "https://console.sakana.ai/models" }, { "name": "τ3 Banking", "score": 20.6, "source": "https://console.sakana.ai/models" }, { "name": "Long Context Reasoning", "score": 73.3, "source": "https://console.sakana.ai/models" }, { "name": "MRCRv2", "score": 93.6, "source": "https://console.sakana.ai/models" }, { "name": "CTI-REALM", "score": 69.4, "source": "https://console.sakana.ai/models" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Quality-first multi-agent model for hard research, analysis, and competitions", "family": "fugu", "limit": { "context": 1000000 }, "links": [ { "label": "Official model catalog", "type": "docs", "url": "https://raw.githubusercontent.com/SakanaAI/fugu/refs/heads/main/configs/files/fugu.json" } ], "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Fugu Ultra", "open_weights": false, "release_date": "2026-06-15", "schema": 2 }, "sakana/fugu-ultra-v2": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Quality-first multi-agent model for hard research, analysis, and competitions", "family": "fugu", "knowledge": "2026-08-28", "limit": { "context": 1000000, "output": 128000 }, "modalities": { "input": [ "text", "image", "application/pdf" ], "output": [ "text" ] }, "name": "Fugu Ultra v2", "open_weights": false, "release_date": "2026-09-11", "schema": 2 }, "sakana/sakana-namazu": { "benchmarks": [ { "name": "AIME26", "score": 96.67, "source": "https://console.sakana.ai/models?model=sakana-namazu" }, { "name": "MMLU-Pro", "score": 90.33, "source": "https://console.sakana.ai/models?model=sakana-namazu" }, { "name": "LiveCodeBench v6", "score": 90.33, "source": "https://console.sakana.ai/models?model=sakana-namazu" }, { "name": "JFBench", "score": 37.4, "source": "https://console.sakana.ai/models?model=sakana-namazu" }, { "name": "Translation", "score": 52.2, "source": "https://console.sakana.ai/models?model=sakana-namazu" }, { "name": "FairPoliticsQA", "score": 56.3, "source": "https://console.sakana.ai/models?model=sakana-namazu" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Japanese-specialized reasoning model based on Kimi K2.6 and tuned for Japanese language, culture, and business workflows", "family": "sakana-namazu", "limit": { "context": 262144, "output": 65536 }, "links": [ { "label": "Official product page", "type": "announcement", "url": "https://sakana.ai/namazu/" }, { "label": "Official model documentation", "type": "docs", "url": "https://console.sakana.ai/models?model=sakana-namazu" } ], "modalities": { "input": [ "text", "image", "application/pdf" ], "output": [ "text" ] }, "name": "Sakana Namazu", "open_weights": false, "release_date": "2026-08-03", "schema": 2 }, "sao10k/l3-lunaris-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "llama", "knowledge": "2023-12-31", "limit": { "context": 8192, "output": 7372 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for this complete merged checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/Sao10K/L3-8B-Lunaris-v1" ], "total_billion": 8.030261248 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3 8B Lunaris", "open_weights": true, "release_date": "2024-08-13", "schema": 2 }, "sao10k/l3.1-euryale-70b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "llama", "knowledge": "2023-12-31", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "70,553,706,496 F16 weight values independently counted from ZIP data.pkl tensor descriptors across all thirty indexed shards. All 723 tensor names match the index; no weight payloads were loaded. This is the v2.2 checkpoint, not the different v2.1 or v2.3 release.", "sources": [ "https://huggingface.co/Sao10K/L3.1-70B-Euryale-v2.2/blob/62b47ff8c2c24d326a7b960773b9c853400150b6/pytorch_model.bin.index.json" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.1 Euryale 70B v2.2", "open_weights": true, "release_date": "2024-08-28", "schema": 2 }, "sao10k/l3.3-euryale-70b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open Llama instruction model for multilingual chat, reasoning, and coding", "family": "llama", "knowledge": "2023-12-31", "limit": { "context": 131072, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for this complete fine-tuned checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/Sao10K/L3.3-70B-Euryale-v2.3" ], "total_billion": 70.553706496 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Llama 3.3 Euryale 70B", "open_weights": true, "release_date": "2024-12-18", "schema": 2 }, "sarvam/sarvam-105b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship Indian-language reasoning model for enterprise multilingual applications", "family": "sarvam", "limit": { "context": 131072, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 10.3, "notes": "Total is the Hub-published F32 count for the complete checkpoint; active count is explicitly reported by the model card.", "sources": [ "https://huggingface.co/api/models/sarvamai/sarvam-105b", "https://huggingface.co/sarvamai/sarvam-105b" ], "total_billion": 106.031767424 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Sarvam 105B", "open_weights": true, "release_date": "2026-02-18", "schema": 2 }, "sarvam/sarvam-30b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient Indian-language reasoning model for chat, coding, and multilingual work", "family": "sarvam", "limit": { "context": 128000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the complete checkpoint. The model card's 2.4B active count excludes embeddings and is not treated as a full per-token count.", "sources": [ "https://huggingface.co/api/models/sarvamai/sarvam-30b" ], "total_billion": 32.152650368 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Sarvam 30B", "open_weights": true, "release_date": "2026-02-18", "schema": 2 }, "sarvam/sarvam-m": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Hybrid-reasoning multilingual model for Indian languages built on Mistral Small", "family": "sarvam", "license": "Apache 2.0", "limit": { "context": 32768, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/sarvamai/sarvam-m" ], "total_billion": 23.5724032 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Sarvam-M", "open_weights": true, "release_date": "2025-05-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sarvamai/sarvam-m" } ] }, "sdaia/allam-2-7b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "ALLaM-2-7b instruction tuned model by SDAIA", "limit": { "context": 4096, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Groq's model card for the exact allam-2-7b identifier explicitly describes an autoregressive transformer with 7 billion parameters. This is its published rounded architecture specification, not inferred from the name or substituted from the separately named ALLaM-7B-Instruct-preview checkpoint. The existing Hugging Face weight URL remains unverified.", "sources": [ "https://console.groq.com/docs/model/allam-2-7b" ], "total_billion": 7 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "ALLaM-2-7b", "open_weights": true, "release_date": "2025-01-23", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/ALLaM-AI/ALLaM-2.0-7B-Instruct" } ] }, "sentence-transformers/all-minilm-l12-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Sentence and short-paragraph embedding model for clustering and semantic search", "license": "Apache-2.0", "limit": { "context": 128, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/sentence-transformers/all-MiniLM-L12-v2" ], "total_billion": 0.03336 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "all-MiniLM-L12-v2", "open_weights": true, "release_date": "2021-08-30", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2" } ] }, "sentence-transformers/all-minilm-l6-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Sentence and short-paragraph embedding model for clustering and semantic search", "license": "Apache-2.0", "limit": { "context": 256, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/sentence-transformers/all-MiniLM-L6-v2" ], "total_billion": 0.022713216 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "all-MiniLM-L6-v2", "open_weights": true, "release_date": "2021-08-30", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2" } ] }, "sentence-transformers/all-mpnet-base-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Sentence and short-paragraph embedding model for clustering and semantic search", "license": "Apache-2.0", "limit": { "context": 384, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2" ], "total_billion": 0.109486464 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "all-mpnet-base-v2", "open_weights": true, "release_date": "2021-08-30", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/all-mpnet-base-v2" } ] }, "sentence-transformers/clip-vit-b-32": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "CLIP model that embeds text and images into a shared vector space", "limit": { "context": 77, "output": 0 }, "modalities": { "input": [ "text", "image" ], "output": [ "embedding" ] }, "name": "CLIP ViT-B-32", "open_weights": true, "release_date": "2021-06-22", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/clip-ViT-B-32" } ] }, "sentence-transformers/clip-vit-b-32-multilingual-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multilingual text encoder that embeds text into the CLIP ViT-B/32 vector space", "license": "Apache-2.0", "limit": { "context": 128, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "CLIP ViT-B-32 Multilingual v1", "open_weights": true, "release_date": "2021-06-22", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/clip-ViT-B-32-multilingual-v1" } ] }, "sentence-transformers/multi-qa-mpnet-base-dot-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Semantic search embedding model trained on 215M question-answer pairs", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/sentence-transformers/multi-qa-mpnet-base-dot-v1" ], "total_billion": 0.109486464 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "multi-qa-mpnet-base-dot-v1", "open_weights": true, "release_date": "2021-08-23", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/multi-qa-mpnet-base-dot-v1" } ] }, "sentence-transformers/paraphrase-minilm-l6-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Sentence embedding model for paraphrase detection and semantic similarity", "license": "Apache-2.0", "limit": { "context": 128, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/sentence-transformers/paraphrase-MiniLM-L6-v2" ], "total_billion": 0.022713216 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "paraphrase-MiniLM-L6-v2", "open_weights": true, "release_date": "2021-05-19", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sentence-transformers/paraphrase-MiniLM-L6-v2" } ] }, "sesame/csm-1b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open conversational speech generation model", "license": "Apache-2.0", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F32 count for the complete CSM checkpoint, not just the nominal 1B language backbone. Alternate Transformers-format files are not summed.", "sources": [ "https://huggingface.co/api/models/sesame/csm-1b" ], "total_billion": 1.552791552 } } }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/pcm" ] }, "name": "CSM 1B", "open_weights": true, "release_date": "2025-03", "schema": 2, "type": "speech", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/sesame/csm-1b" } ] }, "shibing624/text2vec-base-chinese": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Chinese sentence embedding model for semantic similarity and retrieval", "license": "Apache-2.0", "limit": { "context": 128, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "text2vec-base-chinese", "open_weights": true, "release_date": "2022-01-23", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/shibing624/text2vec-base-chinese" } ] }, "sourceful/riverflow-v2-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Fast Riverflow 2.0 image generation and editing model with a reasoning judge", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp" ] }, "name": "Riverflow 2.0 Fast", "open_weights": false, "release_date": "2026-02-02", "schema": 2, "type": "image", "v2_only": true }, "sourceful/riverflow-v2-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Most capable Riverflow 2.0 image generation and editing model with a reasoning judge", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp" ] }, "name": "Riverflow 2.0 Pro", "open_weights": false, "release_date": "2026-02-02", "schema": 2, "type": "image", "v2_only": true }, "sourceful/riverflow-v2.5-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speed-optimized Riverflow 2.5 agentic image model with candidate scoring", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png", "image/jpeg" ] }, "name": "Riverflow 2.5 Fast", "open_weights": false, "release_date": "2026-06-05", "schema": 2, "type": "image", "v2_only": true }, "sourceful/riverflow-v2.5-pro": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Most capable Riverflow 2.5 agentic image model with candidate scoring and 4K output", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp" ], "output": [ "image/webp", "image/png", "image/jpeg" ] }, "name": "Riverflow 2.5 Pro", "open_weights": false, "release_date": "2026-06-05", "schema": 2, "type": "image", "v2_only": true }, "stabilityai/stable-diffusion-xl-base-1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Latent diffusion text-to-image model from Stability AI", "family": "stable-diffusion", "license": "CreativeML Open RAIL++-M", "limit": { "context": 0, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "U-Net", "notes": "Exact stored F32 weight count verified from the component's actual tensor header.", "sources": [ "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/462165984030d82259a11f4367a4eed129e94a7b/unet/diffusion_pytorch_model.safetensors" ], "total_billion": 2.567463684 } ], "notes": "U-Net statistic only; separate text encoders, VAE and duplicate weight formats are not added into a complete model total." } } }, "modalities": { "input": [ "text", "image" ], "output": [ "image" ] }, "name": "Stable Diffusion XL Base 1.0", "open_weights": true, "release_date": "2023-07-26", "schema": 2, "type": "image", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0" } ] }, "stepfun/step-3.5-flash": { "benchmarks": [ { "date": "2026-06-02", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 31.6, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "percent correct", "name": "SciCode", "score": 40.4, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 27.3, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "metric": "resolved", "name": "SWE-Bench Verified", "score": 74.4, "source": "https://arxiv.org/abs/2602.10604" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "StepFun flash lane for quick multimodal reasoning and coding assistance", "family": "step", "knowledge": "2025-01", "last_updated": "2026-02-13", "limit": { "context": 256000, "input": 256000, "output": 256000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 11, "notes": "Full architecture table's 196.81B total includes the 0.81B head; per-token active count is approximate (~11B).", "sources": [ "https://huggingface.co/stepfun-ai/Step-3.5-Flash" ], "total_billion": 196.81 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Step 3.5 Flash", "open_weights": true, "release_date": "2026-01-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" } ] }, "stepfun/step-3.5-flash-2603": { "benchmarks": [ { "date": "2026-06-02", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 34.6, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "percent correct", "name": "SciCode", "score": 38.5, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "date": "2026-06-02", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 32.6, "source": "https://openrouter.ai/stepfun/step-3.5-flash/benchmarks" }, { "dataset": "2,158 text-only questions from the original HLE dataset (Artificial Analysis evaluation)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": "24%", "source": "https://artificialanalysis.ai/models/comparisons/mistral-medium-3-5-vs-step-3-5-flash", "variant": "pass@1; GPT-5.6 Luna (medium) equality checker" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "StepFun flash model for efficient multimodal reasoning, coding, and tool use", "family": "step", "knowledge": "2025-01", "limit": { "context": 256000, "input": 256000, "output": 256000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 11, "notes": "Full architecture table's 196.81B total includes the 0.81B head; per-token active count is approximate (~11B).", "sources": [ "https://huggingface.co/stepfun-ai/Step-3.5-Flash" ], "total_billion": 196.81 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Step 3.5 Flash 2603", "open_weights": true, "release_date": "2026-04-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/stepfun-ai/Step-3.5-Flash" } ] }, "stepfun/step-3.7-flash": { "benchmarks": [ { "date": "2026-05-29", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 56.3, "source": "https://static.stepfun.com/blog/step-3.7-flash/" }, { "date": "2026-05-29", "metric": "resolved", "name": "SWE-Bench Verified", "score": 76.5, "source": "https://static.stepfun.com/blog/step-3.7-flash/" }, { "date": "2026-05-29", "metric": "success rate", "name": "Terminal-Bench", "score": 59.6, "source": "https://static.stepfun.com/blog/step-3.7-flash/", "version": "2.1" }, { "date": "2026-05-29", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 47.2, "source": "https://static.stepfun.com/blog/step-3.7-flash/", "variant": "with tools" }, { "date": "2026-05-29", "metric": "accuracy", "name": "BrowseComp", "score": 75.8, "source": "https://static.stepfun.com/blog/step-3.7-flash/" }, { "date": "2026-05-29", "metric": "success rate", "name": "Toolathlon", "score": 49.5, "source": "https://static.stepfun.com/blog/step-3.7-flash/" }, { "date": "2026-05-29", "metric": "wins or ties", "name": "GDPval", "score": 45.8, "source": "https://static.stepfun.com/blog/step-3.7-flash/" }, { "date": "2026-05-29", "metric": "pass^3", "name": "ClawEval", "score": 67.1, "source": "https://static.stepfun.com/blog/step-3.7-flash/", "version": "1.1" }, { "date": "2026-06-15", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 37.1, "source": "https://openrouter.ai/stepfun/step-3.7-flash/benchmarks" }, { "date": "2026-06-15", "metric": "percent correct", "name": "SciCode", "score": 40.0, "source": "https://openrouter.ai/stepfun/step-3.7-flash/benchmarks" }, { "date": "2026-06-15", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 35.6, "source": "https://openrouter.ai/stepfun/step-3.7-flash/benchmarks" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 856, "source": "https://artificialanalysis.ai/models/comparisons/step-3-7-flash-vs-step-3-5-flash-0202", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Newer StepFun flash model for faster agents, coding, and multimodal prompts", "knowledge": "2026-03-01", "limit": { "context": 256000, "input": 256000, "output": 256000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 11, "notes": "Official rounded 198B full vision-language model size, not the 196B language backbone alone; active count is approximately 11B per token.", "sources": [ "https://huggingface.co/stepfun-ai/Step-3.7-Flash" ], "total_billion": 198 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska" ], "output": [ "text" ] }, "name": "Step 3.7 Flash", "open_weights": true, "release_date": "2026-05-29", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/stepfun-ai/Step-3.7-Flash" } ] }, "stepfun/step-5-preview": { "benchmarks": [ { "effort": "High", "harness": "SWE-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.7, "source": "https://huggingface.co/stepfun-ai/Step-5-Preview", "variant": "temperature=1.0; top_p=0.95", "version": "1.1" }, { "dataset": "all 66 tasks", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 33.3, "source": "https://huggingface.co/datasets/harborframework/terminal-bench?eval_result=stepfun-ai/Step-5-Preview&leaderboard_task_id=terminalbench_4", "version": "4.0.0" }, { "cost": 3.113224, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "all 66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 31.818, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; three full runs, each run scored pass@1; reported ±2.62 percentage points", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 2.4, "source": "https://huggingface.co/stepfun-ai/Step-5-Preview/discussions/10", "variant": "pass@1 averaged over three repeats; reported to one decimal place", "version": "0.1.0" }, { "dataset": "all 70 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 2.86, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single run; pass@1; reported ±2.01 percentage points", "version": "0.1.0" }, { "cost": 2.73409, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "metric": "performance score", "name": "Vals Index", "score": 54.762, "source": "https://www.vals.ai/benchmarks/vals_index", "variant": "reported ±1.16 percentage points", "version": "2.2" }, { "cost": 2.3452525606205166, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 28.5283090216016, "source": "https://artificialanalysis.ai/models/step-5", "version": "5.0.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "StepFun's next-generation flagship base model for coding and professional knowledge work, with native text, image, and video input and a 1M-token context window", "last_updated": "2026-09-20", "limit": { "context": 1000000, "input": 1000000, "output": 1000000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 27, "sources": [ "https://huggingface.co/stepfun-ai/Step-5-Preview" ], "total_billion": 600 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "video/mp4", "video/quicktime", "video/x-matroska" ], "output": [ "text" ] }, "name": "Step 5 Preview", "open_weights": true, "release_date": "2026-09-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/stepfun-ai/Step-5-Preview" } ] }, "surogate/rune-26b-a4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight System One decision model on Gemma 4 26B-A4B that answers up to 256 typed questions about a document with calibrated probabilities in one pass", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete model.safetensors checkpoint also hosted in this repository, not inferred from GGUF file sizes or quantized tensor storage.", "sources": [ "https://huggingface.co/api/models/surogate/rune-26b-a4b-GGUF" ], "total_billion": 25.805936206 } } }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Rune 26B-A4B", "open_weights": true, "release_date": "2026-09-21", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/surogate/rune-26b-a4b-GGUF" } ] }, "swiss-ai/apertus-70b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fully open 70B multilingual LLM supporting 1800+ languages with 65K context. Trained on 15T tokens of compliant open data. Apache 2.0, EU AI Act compliant.", "knowledge": "2025-09", "license": "Apache-2.0", "limit": { "context": 65536, "output": 8192 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2509.14233" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/swiss-ai/Apertus-70B-Instruct-2509" ], "total_billion": 70.59986448 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Apertus 70B", "open_weights": true, "release_date": "2025-09-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/swiss-ai/Apertus-70B-Instruct-2509" } ] }, "swiss-ai/apertus-8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fully open 8B multilingual LLM supporting 1800+ languages with 65K context. Trained on compliant open data. Apache 2.0, EU AI Act compliant.", "knowledge": "2025-09", "license": "Apache-2.0", "limit": { "context": 65536, "output": 8192 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2509.14233" } ], "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 parameter count for the complete unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/swiss-ai/Apertus-8B-Instruct-2509" ], "total_billion": 8.053338176 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Apertus 8B", "open_weights": true, "release_date": "2025-09-02", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509" } ] }, "swiss-ai/apertus-v1.5-70b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fully open multilingual multimodal reasoner from the Swiss AI Initiative with image and audio understanding, an optional thinking mode, and a 262K context window", "license": "Apache-2.0", "limit": { "context": 262144, "output": 8192 }, "modalities": { "input": [ "text", "image", "audio" ], "output": [ "text" ] }, "name": "Apertus 1.5 70B", "open_weights": true, "release_date": "2026-07-24", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/swiss-ai/Apertus-v1.5-70B" } ] }, "swiss-ai/apertus-v1.5-8b": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fully open 8B multilingual multimodal LLM from the Swiss AI Initiative (ETH Zurich, EPFL, CSCS) supporting 1,000+ languages, with weights, training data, and methods fully documented. Apache 2.0.", "license": "Apache-2.0", "limit": { "context": 262144 }, "links": [ { "label": "Paper", "type": "paper", "url": "https://arxiv.org/abs/2610.15542" }, { "label": "Release notes", "type": "blog", "url": "https://huggingface.co/blog/swiss-ai/apertus-v15-release-notes" } ], "modalities": { "input": [ "text", "image/jpeg", "image/png", "audio/mpeg" ], "output": [ "text" ] }, "name": "Apertus v1.5 8B", "open_weights": true, "release_date": "2026-10-15", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/swiss-ai/Apertus-v1.5-8B" } ] }, "tencent/hy-image-v3.5-preview": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Preview image generation and multi-turn editing model", "family": "Hy", "limit": { "context": 100000, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "image/png" ] }, "name": "Hy Image 3.5 Preview", "open_weights": false, "release_date": "2026-09-22", "schema": 2, "type": "image", "v2_only": true }, "tencent/hy-mt2-1.8b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multilingual translation model for complex real-world tasks, supporting translation among 33 languages and multilingual translation instructions", "family": "Hy", "limit": { "context": 8192, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint including embeddings, not the nominal size designation.", "sources": [ "https://huggingface.co/api/models/tencent/Hy-MT2-1.8B" ], "total_billion": 2.038515712 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy-MT2-1.8B", "open_weights": true, "release_date": "2026-05-21", "schema": 2 }, "tencent/hy-mt2-30b-a3b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multilingual translation model for complex real-world tasks, supporting translation among 33 languages and multilingual translation instructions", "family": "Hy", "limit": { "context": 8192, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete unquantized checkpoint. Active count not inferred from the model-name designation.", "sources": [ "https://huggingface.co/api/models/tencent/Hy-MT2-30B-A3B" ], "total_billion": 30.064725888 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy-MT2-30B-A3B", "open_weights": true, "release_date": "2026-05-21", "schema": 2 }, "tencent/hy-mt2-7b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multilingual translation model for complex real-world tasks, supporting translation among 33 languages and multilingual translation instructions", "family": "Hy", "limit": { "context": 8192, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete checkpoint including embeddings, not the nominal size designation.", "sources": [ "https://huggingface.co/api/models/tencent/Hy-MT2-7B" ], "total_billion": 8.029540352 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy-MT2-7B", "open_weights": true, "release_date": "2026-05-21", "schema": 2 }, "tencent/hy3": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 78, "source": "https://huggingface.co/tencent/Hy3" }, { "harness": "SWE-agent", "metric": "score", "name": "SWE-Bench Multilingual", "score": 75.8, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "harness": "SWE-agent", "metric": "score", "name": "SWE-Bench Pro", "score": 57.9, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "harness": "Terminus 2", "metric": "score", "name": "Terminal-Bench", "score": 71.7, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; 4h timeout; 500 episodes", "version": "2.1" }, { "harness": "Claude Code", "metric": "score", "name": "NL2Repo", "score": 45.6, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; 250 turns; 12000s timeout" }, { "harness": "mini-swe-agent", "metric": "score", "name": "DeepSWE", "score": 28, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; 2h timeout" }, { "harness": "Tencent internal search harness", "metric": "score", "name": "BrowseComp", "score": 84.2, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "harness": "Tencent internal search harness", "metric": "score", "name": "WideSearch", "score": 76.4, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "harness": "Tencent internal search harness", "metric": "score", "name": "DeepSearchQA", "score": 91, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "dataset": "500 public tasks", "harness": "Scale AI", "metric": "score", "name": "MCP Atlas", "score": 79.1, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; April 2026; 100 tool calls" }, { "metric": "score", "name": "Toolathlon", "score": 48.5, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "pass@1", "name": "APEX-Agents", "score": 25.6, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "dataset": "105 queries", "harness": "Tencent internal harness", "metric": "pass@3", "name": "ClawEval", "score": 68.5, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort", "version": "20260325" }, { "dataset": "35 text-only tasks", "harness": "OpenClaw", "metric": "score", "name": "WildClawBench", "score": 53.6, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "dataset": "79 text-only tasks", "harness": "Claude Code", "metric": "average over 3 runs", "name": "SkillsBench", "score": 55.3, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "dataset": "text-only", "metric": "score", "name": "Humanity's Last Exam", "score": 53.2, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; with tools" }, { "dataset": "text-only", "metric": "score", "name": "Humanity's Last Exam", "score": 37, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; without tools" }, { "metric": "score", "name": "GPQA Diamond", "score": 90.4, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "FrontierScience", "score": 21.3, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; research" }, { "metric": "score", "name": "FrontierScience", "score": 74.8, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort; olympiad" }, { "metric": "score", "name": "USAMO", "score": 72, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort", "version": "2026" }, { "metric": "score", "name": "MathArena Apex", "score": 38.7, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "ArxivMath", "score": 52.2, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "pass@12", "name": "HorizonMath", "score": 7.1, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "PHYBench", "score": 77.4, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "CMT Benchmark", "score": 37.8, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "IMOAnswerBench", "score": 90, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "SuperChem", "score": 54.9, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "CL-bench", "score": 23.8, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "CL-bench-life", "score": 17, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "metric": "score", "name": "AA-LCR", "score": 73.4, "source": "https://huggingface.co/tencent/Hy3", "variant": "highest reasoning effort" }, { "dataset": "66 tasks", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1, "source": "https://artificialanalysis.ai/models/comparisons/hy3-non-reasoning-vs-hy3", "variant": "pass@1; average of 3 repeats per task; maximum 500 agent steps; upstream task timeouts", "version": "4.0.0" }, { "dataset": "100 tasks", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 1.2, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "20 trials per task; no tools" }, { "dataset": "May 2025 revision; 2,158 text-only questions from 2,500 total", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 33, "source": "https://artificialanalysis.ai/models/comparisons/hy3-non-reasoning-vs-hy3", "variant": "pass@1; judge=GPT-5.6 Luna (medium)" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1067, "source": "https://artificialanalysis.ai/models/comparisons/hy3-non-reasoning-vs-hy3", "variant": "blind pairwise grading; Elo anchored to DeepSeek V4.1 Flash (max)=1600", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Tencent Hy reasoning model for coding, instruction following, and agent tasks", "family": "Hy", "limit": { "context": 256000, "input": 192000, "output": 128000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 21, "notes": "Official table's total and active counts; the card lists 3.8B MTP parameters separately. No extra component sum inferred.", "sources": [ "https://huggingface.co/tencent/Hy3" ], "total_billion": 295 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy3", "open_weights": true, "release_date": "2026-07-06", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/tencent/Hy3" } ] }, "tencent/hy3-preview": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 74.4, "source": "https://huggingface.co/tencent/Hy3-preview" }, { "dataset": "cais/hle", "name": "Humanity's Last Exam", "score": "30*", "source": "https://huggingface.co/tencent/Hy3-preview" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Tencent Hy reasoning model for coding, instruction following, and agent tasks", "family": "Hy", "limit": { "context": 256000, "output": 64000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 21, "notes": "Official table's total and active counts; the card lists 3.8B MTP parameters separately. No extra component sum inferred.", "sources": [ "https://huggingface.co/tencent/Hy3-preview" ], "total_billion": 295 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy3 preview", "open_weights": true, "release_date": "2026-04-20", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/tencent/Hy3-preview" } ] }, "tencent/hy4": { "benchmarks": [ { "cost": 1.0311060706397974, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 34.5552385392508, "source": "https://artificialanalysis.ai/models/tencent-hy4", "version": "5.0.1" }, { "dataset": "113 tasks", "metric": "pass@1", "name": "DeepSWE", "score": 74.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "1.1" }, { "metric": "score", "name": "SWE-Marathon", "score": 43.8, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "1.1" }, { "harness": "Claude Code", "metric": "score", "name": "ProgramBench", "score": 30.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "Almost Solved; highest available reasoning setting" }, { "harness": "Claude Code", "metric": "score", "name": "Terminal-Bench", "score": 44.4, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting; up to 500 turns; 13-hour task limit", "version": "4.0.0" }, { "metric": "score", "name": "FrontierCode", "score": 31.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "metric": "score", "name": "Hy-CompanyBench (internal)", "score": 78.8, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "metric": "score", "name": "Hy-SWE Max Verified (internal)", "score": 45.4, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "metric": "score", "name": "Hy-Backend (internal)", "score": 45.1, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "3.0" }, { "metric": "score", "name": "WebCraftBench (internal)", "score": 78.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "GameFix Bench (internal)", "score": 68.4, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "metric": "score", "name": "3DArena-Modeling (internal)", "score": 37.1, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "Goal2Repo (internal)", "score": 23.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "AutoCatBench (internal)", "score": 57.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "Hy-LifeSearch (internal)", "score": 60.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "Hy-BrowseComp (internal)", "score": 59.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "harness": "Official APEX harness", "metric": "pass@1", "name": "APEX-Agents", "score": 75.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting; up to 250 steps per trial", "version": "1.1" }, { "metric": "Pass Rate", "name": "Agents' Last Exam", "score": 41.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "1.1" }, { "dataset": "600 public tasks", "metric": "task_completed_correctly", "name": "AutomationBench", "score": 56.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "1.0.6" }, { "metric": "score", "name": "E-Bench (internal)", "score": 70.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.0" }, { "metric": "score", "name": "Agents' First Exam-IF (internal)", "score": 79.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "Tag-bench (internal)", "score": 48.8, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "PPTGen-Bench (internal)", "score": 60.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "dataset": "HLE-Full", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 59.1, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "with tools; highest available reasoning setting" }, { "metric": "score", "name": "Hy-FrontierMath T5 (internal)", "score": 13.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "metric": "score", "name": "ProofBench (internal)", "score": 5.5, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "harness": "Claude Code", "metric": "score", "name": "Terminal-Bench-Science", "score": 19.8, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "up to 500 turns; 13-hour task limit; highest available reasoning setting", "version": "0.1.0" }, { "metric": "score", "name": "PostTrainBench", "score": 35.2, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "1.2" }, { "metric": "score", "name": "Foundation Model Bench (internal)", "score": 28.1, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "dataset": "123 executable tasks across seven disciplines", "metric": "score", "name": "TenureTrail (internal)", "score": 53.6, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "dataset": "Full suite", "metric": "partial score", "name": "OSWorld", "score": 78.9, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting", "version": "2.1" }, { "metric": "score", "name": "MMMU-Pro", "score": 78.6, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "highest available reasoning setting" }, { "harness": "Jupyter notebook", "metric": "pass@5", "name": "ZeroBench-main", "score": 55.4, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "with tools; 10 runs per problem; up to 80 tool rounds; highest available reasoning setting" }, { "metric": "score", "name": "BuddyVQA Atlas (internal)", "score": 81.0, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "with tools; highest available reasoning setting" }, { "metric": "score", "name": "BuddyVQA Atlas (internal)", "score": 55.6, "source": "https://hy.tencent.ai/research/hy4?langVersion=en", "variant": "without tools; highest available reasoning setting" }, { "cost": 1.720395, "cost_unit": "USD/test", "metric": "performance score", "name": "Vals Index", "score": 59.353, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "version": "2.2" }, { "cost": 1.280113, "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 39.394, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 32.7, "source": "https://artificialanalysis.ai/models/hy4", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "A next-generation multimodal productivity model with significantly enhanced Agent and complex task execution capabilities.", "family": "Hy", "limit": { "context": 1048576, "input": 983010, "output": 65536 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "image/gif" ], "output": [ "text" ] }, "name": "Hy4", "open_weights": false, "release_date": "2026-11-04", "schema": 2 }, "tencent/hy4-preview": { "benchmarks": [ { "cost": 3.07266, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Full (66 tasks)", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 8.081, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Full (70 tasks)", "harness": "Terminus 2", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 1.43, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single pass@1 run; infrastructure/provider errors retried; 8-hour task limit", "version": "0.1.0" }, { "dataset": "100 expert tasks", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 1.6, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge", "version": "unspecified" }, { "dataset": "Text-only", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.4, "source": "https://huggingface.co/tencent/Hy4-preview", "variant": "no tools" }, { "dataset": "Text-only", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 55.4, "source": "https://huggingface.co/tencent/Hy4-preview", "variant": "with tools" }, { "cost": 1.56824, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "metric": "performance score", "name": "Vals Index", "score": 53.209, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "A next-generation productivity model with significantly enhanced Agent and complex task execution capabilities.", "family": "Hy", "limit": { "context": 1024000, "output": 64000 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 49, "sources": [ "https://huggingface.co/tencent/Hy4-preview" ], "total_billion": 770 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Hy4 preview", "open_weights": true, "release_date": "2026-08-28", "schema": 2 }, "tencent/kinfra-text-embedding-0.6b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lightweight Tencent text embedding model producing 1,024-dimensional vectors for large-scale retrieval", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Kinfra Text Embedding 0.6B", "open_weights": false, "release_date": "2026-07", "schema": 2, "type": "embedding", "v2_only": true }, "tencent/kinfra-text-embedding-4b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Tencent text embedding model producing 2,560-dimensional vectors for high-quality retrieval", "limit": { "context": 32768, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Kinfra Text Embedding 4B", "open_weights": false, "release_date": "2026-07", "schema": 2, "type": "embedding", "v2_only": true }, "thedrummer/cydonia-24b-v4.1": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight instruction model for adaptable chat and self-hosted production workloads", "knowledge": "2024-04-30", "limit": { "context": 131072, "output": 117964 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete version-specific unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/TheDrummer/Cydonia-24B-v4.1" ], "total_billion": 23.5724032 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Cydonia 24B V4.1", "open_weights": true, "release_date": "2025-09-27", "schema": 2 }, "thedrummer/skyfall-36b-v2": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight instruction model for adaptable chat and self-hosted production workloads", "knowledge": "2024-06-30", "limit": { "context": 32768, "output": 29491 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete version-specific unquantized checkpoint, converted to B.", "sources": [ "https://huggingface.co/api/models/TheDrummer/Skyfall-36B-v2" ], "total_billion": 36.91053568 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Skyfall 36B V2", "open_weights": true, "release_date": "2025-03-10", "schema": 2 }, "thedrummer/unslopnemo-12b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight instruction model for adaptable chat and self-hosted production workloads", "knowledge": "2024-04-30", "limit": { "context": 1024000, "output": 819200 }, "metadata": { "modelatlas": { "parameters": { "notes": "OpenRouter explicitly maps thedrummer/unslopnemo-12b to version v4.1. Uses that complete BF16 checkpoint's published count, not the distinct v1 checkpoint.", "sources": [ "https://huggingface.co/api/models/TheDrummer/UnslopNemo-12B-v4.1", "https://openrouter.ai/api/v1/models" ], "total_billion": 12.2477824 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "UnslopNemo 12B", "open_weights": true, "release_date": "2024-11-08", "schema": 2 }, "thenlper/gte-base": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model producing 768-dimensional vectors", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/thenlper/gte-base" ], "total_billion": 0.10948224 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "GTE Base", "open_weights": true, "release_date": "2023-07", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/thenlper/gte-base" } ] }, "thenlper/gte-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model producing 1024-dimensional vectors", "license": "MIT", "limit": { "context": 512, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published F16 parameter count for the complete checkpoint; excludes separately reported integer tensors.", "sources": [ "https://huggingface.co/api/models/thenlper/gte-large" ], "total_billion": 0.335141888 } } }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "GTE Large", "open_weights": true, "release_date": "2023-07", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/thenlper/gte-large" } ] }, "thinkingmachines/inkling": { "benchmarks": [ { "dataset": "SWE-bench Pro V2 Full public set (642 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 89.88, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full", "variant": "reported ±2.10; uncapped cost limit; 250-turn limit", "version": "2.0.0" }, { "cost": 2.5820625258588943, "cost_source": "https://artificialanalysis.ai/models/inkling", "cost_unit": "USD/task", "dataset": "Terminal-Bench 4.0 (66 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1, "source": "https://artificialanalysis.ai/models/comparisons/inkling-vs-gpt-5-6-sol-xhigh", "variant": "AA independent evaluation; pass@1 averaged over three repeats per task; 500-step cap", "version": "4.0.0" }, { "cost": 2.495998, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench 4.0 (66 tasks)", "effort": "0.99", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0.505, "source": "https://vals-ai.com/models/thinkingmachines_inkling", "variant": "Vals AI avg@3; pass@1 per run; reported ±0.51; 8-hour task timeout", "version": "4.0.0" }, { "dataset": "Terminal-Bench-Science 0.1.0 (70 tasks)", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 0, "source": "https://artificialanalysis.ai/models/comparisons/inkling-vs-gpt-5-6-sol-xhigh", "variant": "AA independent evaluation; pass@1 averaged over three repeats per task", "version": "0.1.0" }, { "dataset": "Private held-out split (657 tasks across six business domains)", "effort": "Xhigh", "metric": "AutomationBench-AA Score", "name": "AutomationBench", "score": 5, "source": "https://artificialanalysis.ai/models/comparisons/inkling-vs-gpt-5-6-sol-xhigh", "variant": "AA headline score: zero for any guardrail violation, otherwise share of objectives completed; one run per task; 50-turn cap", "version": "1.0.6" }, { "dataset": "Chartography full set (100 tasks)", "effort": "High reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 10, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Official leaderboard entry; run count not reported; paper says 20 trials per task while current harness README says 10 runs" }, { "dataset": "HLE text-only subset (2,158 questions)", "effort": "Xhigh", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 32, "source": "https://artificialanalysis.ai/models/comparisons/inkling-vs-gpt-5-6-sol-xhigh", "variant": "Artificial Analysis independent evaluation; text-only, no tools" }, { "cost": 1.3519575079737436, "cost_unit": "USD/weighted task", "dataset": "Nine-evaluation composite", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 12.4781698715401, "source": "https://artificialanalysis.ai/models/inkling", "version": "5.0.1" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1080, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "reported interval −22/+22; shell and web browsing", "version": "2.1" }, { "cost": 1.847767, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "GDP-weighted finance, coding, legal, and tax evaluations", "effort": "0.99", "metric": "performance score", "name": "Vals Index", "score": 28.961, "source": "https://www.vals.ai/models/thinkingmachines_inkling", "variant": "reported ±1.12; Inkling via Thinking Machines", "version": "2.2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal MoE reasoning model (975B total, 41B active) for text, image, and audio", "family": "ling", "license": "Apache-2.0", "limit": { "context": 1048576, "output": 1048576 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 41, "sources": [ "https://huggingface.co/thinkingmachines/Inkling" ], "total_billion": 975 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Inkling", "open_weights": true, "release_date": "2026-07-15", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/thinkingmachines/Inkling" } ] }, "thinkingmachines/inkling-2-small": { "benchmarks": [ { "cost": 3.509318, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.Bndh1wJE.json", "cost_unit": "USD/test", "effort": "0.99", "metric": "performance score", "name": "Vals Index", "score": 56.081, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 8.218043, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.CfJg38mV.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "0.99", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 12.626, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "cost": 1.1784281349617984, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 30.6286781498958, "source": "https://artificialanalysis.ai/models/inkling-2-small", "version": "5.0.1" }, { "effort": "0.99", "metric": "pass@1", "name": "DeepSWE", "score": 72.1, "source": "https://thinkingmachines.ai/news/inkling-2-small/", "variant": "Publisher-reported Datacurve result; temperature 1.0; 1M-token maximum trajectory limit", "version": "1.1" }, { "dataset": "SWE-bench Pro V2 Hard", "effort": "0.99", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 86.3, "source": "https://thinkingmachines.ai/model-card/inkling-2-small/", "variant": "Publisher-reported Scale AI result; temperature 1.0; 1M-token maximum trajectory limit", "version": "2.0.0" }, { "dataset": "SWE-bench Pro V2 Full", "effort": "0.99", "metric": "task resolution rate", "name": "SWE-bench Pro", "score": 97.7, "source": "https://thinkingmachines.ai/model-card/inkling-2-small/", "variant": "Publisher-reported Scale AI result; temperature 1.0; 1M-token maximum trajectory limit", "version": "2.0.0" }, { "cost": 2.4766515872666672, "cost_unit": "USD/task", "dataset": "Terminal-Bench 4.0 (66 tasks)", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 13.1313131313131, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "AA independent evaluation; pass@1 averaged over three repeats per task; 500-step cap; no context compaction or summarization", "version": "4.0.0" }, { "cost": 3.6782952861271414, "cost_unit": "USD/task", "dataset": "Terminal-Bench-Science 0.1.0 (70 tasks)", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 5.71428571428571, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "AA independent evaluation; pass@1 averaged over three repeats per task; 1,000-step cap", "version": "0.1.0" }, { "cost": 5.546533, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_science.KYAu-wfB.json", "cost_unit": "USD/test", "dataset": "Terminal-Bench-Science 0.1.0 (70 tasks)", "effort": "0.99", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 2.857, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "Vals independent evaluation; single-run pass@1; infrastructure/provider-error retries only; 8-hour task limit", "version": "0.1.0" }, { "cost": 0.058102370538660575, "cost_unit": "USD/task", "dataset": "Private held-out split (657 tasks across six business domains)", "metric": "AutomationBench-AA Score", "name": "AutomationBench-AA", "score": 45.621320178972645, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "AA independent evaluation; zero for any guardrail violation, otherwise share of objectives completed; one run per task; 50-turn cap", "version": "1.0.6" }, { "effort": "0.99", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 36.7, "source": "https://thinkingmachines.ai/model-card/inkling-2-small/", "variant": "Publisher-reported snapshot; original (without Python); temperature 1.0; 1M-token maximum trajectory limit; run count not disclosed" }, { "effort": "0.99", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 80.4, "source": "https://thinkingmachines.ai/model-card/inkling-2-small/", "variant": "Publisher-reported snapshot; with Python; temperature 1.0; 1M-token maximum trajectory limit; run count not disclosed" }, { "dataset": "HLE text-only subset (2,158 questions)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 44.114921223355, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "AA independent evaluation; text-only; no tools" }, { "dataset": "GDPval public set (220 tasks across 44 occupations)", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1630.23, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "AA independent evaluation; reported interval [1609, 1652]; shell and web browsing", "version": "2.1" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 28.7, "source": "https://artificialanalysis.ai/models/inkling-2-small", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "tools": { "supported": true } }, "description": "Thinking Machines' 594B-total, 17B-active MoE reasoning model for text and image inputs, text output, coding, and tool use; weights are initially closed pending safety testing.", "limit": {}, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Inkling 2 Small", "open_weights": false, "release_date": "2026-11-05", "schema": 2 }, "thinkingmachines/inkling-small": { "benchmarks": [ { "cost": 0.321679, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "0.99", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; per-run pass@1", "version": "4.0.0" }, { "dataset": "100 tasks", "effort": "xHigh reasoning", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 9.1, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task" }, { "dataset": "2,158 text-only questions", "effort": "0.99", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 31.6, "source": "https://thinkingmachines.ai/news/inkling-small/", "variant": "text-only; no tools" }, { "effort": "0.99", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 47.8, "source": "https://thinkingmachines.ai/news/inkling-small/", "variant": "with tools" }, { "cost": 0.20316116043899832, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 12.9101620926084, "source": "https://artificialanalysis.ai/models/inkling-small/", "version": "5.0.1" }, { "cost": 0.401307, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "0.99", "metric": "performance score", "name": "Vals Index", "score": 27.588, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Multimodal MoE reasoning model (276B total, 12B active) for text, image, and audio", "family": "ling", "license": "Apache-2.0", "limit": { "context": 1048576, "output": 1048576 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "sources": [ "https://huggingface.co/thinkingmachines/Inkling-Small" ], "total_billion": 276 } } }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "audio/wav" ], "output": [ "text" ] }, "name": "Inkling Small", "open_weights": true, "release_date": "2026-07-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/thinkingmachines/Inkling-Small" } ] }, "trendyol/asure-12b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Turkish-language multimodal instruct model built on Gemma 3 12B for e-commerce text, chat, and image-text tasks", "family": "gemma", "last_updated": "2026-02-20", "license": "Gemma", "limit": { "context": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Official model card's rounded multimodal model size.", "sources": [ "https://huggingface.co/Trendyol/Trendyol-LLM-Asure-12B" ], "total_billion": 12 } } }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Trendyol Asure 12B", "open_weights": true, "release_date": "2026-02-19", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/Trendyol/Trendyol-LLM-Asure-12B" } ] }, "turion/turion-extract-alpha": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight extraction model on Gemma 4 12B (LoRA plus extraction heads) that returns one typed value or null per requested field from a text document in one forward pass", "license": "Apache-2.0", "limit": { "context": 16384, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Turion Extract alpha", "open_weights": true, "release_date": "2026-10-25", "schema": 2, "type": "extraction", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/turion-ai/turion-extract-alpha" } ] }, "turion/turion-jev-31b": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Open-weight System One decision model fine-tuned from Gemma 4 31B that returns probabilities for typed choice, yes/no and score questions about text or JSON", "license": "Apache-2.0", "limit": { "context": 16384, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "decision" ] }, "name": "Turion Jev 31B", "open_weights": true, "release_date": "2026-10-24", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/turion-ai/turion-jev-31b" } ] }, "typesafe/jev-1.13.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "System One model for fast, typed probabilistic decisions over text or structured state", "limit": { "context": 64000, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "decision" ] }, "name": "Jev 1.13", "open_weights": false, "release_date": "2026-09-15", "schema": 2, "type": "decision" }, "typesafe/jev-1.14.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "System One model for fast, typed probabilistic decisions over text or structured state", "limit": { "context": 64000, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "decision" ] }, "name": "Jev 1.14", "open_weights": false, "release_date": "2026-10-14", "schema": 2, "type": "decision" }, "typesafe/jev-latest": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "System One model for fast, typed probabilistic decisions over text or structured state", "limit": { "context": 64000, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "decision" ] }, "name": "Jev", "open_weights": false, "release_date": "2026-09-15", "schema": 2, "type": "decision" }, "unbiased/pareto": { "benchmarks": [ { "dataset": "30-task slice; not the full 113-task v1.1 set", "date": "2026-09-20", "metric": "score (%)", "name": "DeepSWE", "score": 70.0, "source": "https://unbiased.ai/model-card/", "variant": "Pareto 26.9; preliminary Unbiased run", "version": "1.1" }, { "dataset": "task denominator not reported", "date": "2026-10-01", "metric": "score (%)", "name": "DeepSWE", "score": 69.9, "source": "https://unbiased.ai/blog/pareto-26-10-preview/", "variant": "Pareto 26.10 preview; preliminary October 1 run", "version": "1.1" }, { "date": "2026-10-01", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 50.8, "source": "https://unbiased.ai/blog/pareto-26-10-preview/", "variant": "Pareto 26.10 preview; preliminary October 1 run", "version": "4.0.0" }, { "dataset": "HLE text-only; item count not reported", "date": "2026-10-01", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 49.9, "source": "https://unbiased.ai/blog/pareto-26-10-preview/", "variant": "Pareto 26.10 preview; preliminary October 1 run", "version": "unspecified" } ], "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Blended multimodal model that runs multiple language models in parallel and returns a synthesized answer", "limit": { "context": 262144, "output": 131072 }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Pareto", "open_weights": false, "release_date": "2026-09-17", "schema": 2 }, "unbiased/pareto-26.10": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Blended multimodal model that selects frontier and open models for each request and blends their outputs into one answer", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Pareto 26.10", "open_weights": false, "release_date": "2026-10-15", "schema": 2 }, "undi95/remm-slerp-l2-13b": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Open-weight instruction model for adaptable chat and self-hosted production workloads", "knowledge": "2023-06-30", "limit": { "context": 6144, "output": 5529 }, "metadata": { "modelatlas": { "parameters": { "notes": "13,015,864,320 F32 weight values independently counted from ZIP data.pkl tensor descriptors across all three indexed shards. All 363 tensor names match the index; no weight payloads were loaded. This is a direct checkpoint count, not the historical leaderboard's rounded 13B size class.", "sources": [ "https://huggingface.co/undi95/ReMM-SLERP-L2-13B/blob/9cf491450795628e28b8fd73f411977f7908c623/pytorch_model.bin.index.json" ], "total_billion": 13.01586432 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "ReMM SLERP 13B", "open_weights": true, "release_date": "2023-07-22", "schema": 2 }, "upstage/solar-10.7b-instruct-v1.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Upstage depth-upscaled 10.7B instruction model for single-turn chat", "family": "solar", "license": "CC-BY-NC-4.0", "limit": { "context": 4096, "output": 4096 }, "metadata": { "modelatlas": { "parameters": { "sources": [ "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0" ], "total_billion": 10.7 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "SOLAR 10.7B Instruct v1.0", "open_weights": true, "release_date": "2023-12-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0" } ] }, "upstage/solar-mini4": { "benchmarks": [ { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 1.0, "source": "https://artificialanalysis.ai/articles/korean-ai-lab-upstage-releases-solar-mini-4", "version": "4.0.0" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1072, "source": "https://artificialanalysis.ai/articles/korean-ai-lab-upstage-releases-solar-mini-4", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Upstage's compact, cost-efficient model for agentic use", "family": "solar-mini", "knowledge": "2026-02", "limit": { "context": 524288, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Solar Mini 4", "open_weights": false, "release_date": "2026-09-22", "schema": 2 }, "upstage/solar-pro2": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Flagship model for demanding analysis, coding, and production agent workflows", "family": "solar-pro", "knowledge": "2025-03", "limit": { "context": 65536, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Solar Pro 2", "open_weights": false, "release_date": "2025-05-20", "schema": 2 }, "upstage/solar-pro3": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Flagship model for demanding analysis, coding, and production agent workflows", "family": "solar-pro", "knowledge": "2025-07", "last_updated": "2026-03-23", "limit": { "context": 131072, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Solar Pro 3", "open_weights": false, "release_date": "2026-01-26", "schema": 2 }, "upstage/solar-pro4": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 29, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-astra-high-vs-solarpro4" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Upstage's flagship model, specialized for agentic use", "family": "solar-pro", "knowledge": "2026-02", "limit": { "context": 524288, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Solar Pro 4", "open_weights": false, "release_date": "2026-08-06", "schema": 2 }, "vispark/vision-large": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Most capable Vision model for complex reasoning, detailed media analysis, and structured output over a 1M-token context window.", "last_updated": "2026-09", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image", "audio", "video", "application/pdf" ], "output": [ "text" ] }, "name": "Vision Large", "open_weights": false, "release_date": "2024-05-15", "schema": 2 }, "vispark/vision-medium": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Balanced multimodal model pairing a 1M-token context window with deeper reasoning for analysis, content creation, and tool use across text, image, audio, video, and PDF inputs.", "last_updated": "2026-09", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image", "audio", "video", "application/pdf" ], "output": [ "text" ] }, "name": "Vision Medium", "open_weights": false, "release_date": "2024-05-15", "schema": 2 }, "vispark/vision-small": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast, low-cost multimodal model for understanding text, images, audio, video, and PDFs, with tool calling and a 1M-token context window.", "last_updated": "2026-09", "limit": { "context": 1000000, "output": 65536 }, "modalities": { "input": [ "text", "image", "audio", "video", "application/pdf" ], "output": [ "text" ] }, "name": "Vision Small", "open_weights": false, "release_date": "2024-05-15", "schema": 2 }, "vivgrid/viv-fast": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Fast coding model", "knowledge": "2026-09", "limit": { "context": 1000000, "output": 256000 }, "links": [ { "label": "Model docs", "url": "https://docs.vivgrid.com/models/viv-fast" } ], "modalities": { "input": [ "text", "image" ], "output": [ "text" ] }, "name": "Viv Fast", "open_weights": false, "release_date": "2026-09-21", "schema": 2 }, "vllm-sr/d3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "vLLM Semantic Router 27B decision model that answers typed choice, yes/no and score questions about text, images or video with probabilities", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Includes the 0.46B vision encoder, as explicitly reported in the model card.", "sources": [ "https://huggingface.co/vllm-sr/d3" ], "total_billion": 26.09 } } }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "decision" ] }, "name": "d3", "open_weights": true, "release_date": "2026-10-10", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/vllm-sr/d3" } ] }, "vllm-sr/d3-flash": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "vLLM Semantic Router 9B decision model that answers typed choice, yes/no and score questions about text, images or video with probabilities", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "notes": "Includes the 0.46B vision encoder, as explicitly reported in the model card.", "sources": [ "https://huggingface.co/vllm-sr/d3-flash" ], "total_billion": 8.39 } } }, "modalities": { "input": [ "text", "image", "video" ], "output": [ "decision" ] }, "name": "d3-flash", "open_weights": true, "release_date": "2026-10-10", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/vllm-sr/d3-flash" } ] }, "vllm-sr/d3-mini": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "vLLM Semantic Router 4B decision model that answers typed choice, yes/no and score questions about text, images or video with probabilities", "license": "Apache-2.0", "limit": { "context": 262144, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "video/mp4", "video/webm", "video/quicktime", "video/x-matroska" ], "output": [ "decision" ] }, "name": "d3-mini", "open_weights": true, "release_date": "2026-10-10", "schema": 2, "type": "decision", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/vllm-sr/d3-mini" } ] }, "voyageai/rerank-2.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Instruction-following reranker optimized for retrieval quality", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "rerank-2.5", "open_weights": false, "release_date": "2025-08-11", "schema": 2, "type": "reranking", "v2_only": true }, "voyageai/rerank-2.5-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Instruction-following reranker optimized for latency and quality", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "rerank-2.5-lite", "open_weights": false, "release_date": "2025-08-11", "schema": 2, "type": "reranking", "v2_only": true }, "voyageai/rerank-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Reranker optimized for retrieval quality, successor to rerank-2.5", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "rerank-3", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "reranking", "v2_only": true }, "voyageai/rerank-3-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Reranker optimized for latency and cost, successor to rerank-2.5-lite", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "reranking" ] }, "name": "rerank-3-lite", "open_weights": false, "release_date": "2026-09-30", "schema": 2, "type": "reranking", "v2_only": true }, "voyageai/voyage-4": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose and multilingual embedding model balancing retrieval quality and cost", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "voyage-4", "open_weights": false, "release_date": "2026-01-15", "schema": 2, "type": "embedding", "v2_only": true }, "voyageai/voyage-4-large": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "General-purpose and multilingual embedding model optimized for retrieval quality", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "voyage-4-large", "open_weights": false, "release_date": "2026-01-15", "schema": 2, "type": "embedding", "v2_only": true }, "voyageai/voyage-4-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Lightweight general-purpose embedding model optimized for latency and cost", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "voyage-4-lite", "open_weights": false, "release_date": "2026-01-15", "schema": 2, "type": "embedding", "v2_only": true }, "voyageai/voyage-code-4": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Code embedding model for code retrieval and coding agents", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "voyage-code-4", "open_weights": false, "release_date": "2026-08-13", "schema": 2, "type": "embedding", "v2_only": true }, "voyageai/voyage-multimodal-3.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Multimodal embedding model for text, images, video, and interleaved content", "family": "voyage", "limit": { "context": 32000, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg", "image/webp", "image/gif", "video/mp4" ], "output": [ "embedding" ] }, "name": "voyage-multimodal-3.5", "open_weights": false, "release_date": "2026-01-15", "schema": 2, "type": "embedding", "v2_only": true }, "whereisai/uae-large-v1": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "English text embedding model trained with angle-optimized contrastive learning", "license": "MIT", "limit": { "context": 512, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "UAE Large V1", "open_weights": true, "release_date": "2023-12-04", "schema": 2, "type": "embedding", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/WhereIsAI/UAE-Large-V1" } ] }, "writer/palmyra-x4": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Enterprise language model for workflow automation, coding, data analysis, and tool use", "family": "palmyra", "last_updated": "2025-04-28", "limit": { "context": 128000, "output": 4096 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "Palmyra X4", "open_weights": false, "release_date": "2024-10-09", "schema": 2 }, "writer/palmyra-x5": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Reasoning model for deliberate analysis, multi-step problem solving, and tool use", "family": "palmyra", "limit": { "context": 1000000, "output": 8192 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif" ], "output": [ "text" ] }, "name": "Palmyra X5", "open_weights": false, "release_date": "2025-04-28", "schema": 2 }, "xai/grok-4-fast-non-reasoning": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Grok model for responsive chat, reasoning, and tool-assisted work", "family": "grok", "limit": { "context": 2000000, "output": 2000000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "audio", "video" ], "output": [ "text" ] }, "name": "X-Ai/Grok-4-Fast-Non-Reasoning", "open_weights": false, "release_date": "2025-12-18", "schema": 2 }, "xai/grok-4-fast-reasoning": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Grok model for responsive chat, reasoning, and tool-assisted work", "family": "grok", "limit": { "context": 2000000, "output": 2000000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "audio", "video" ], "output": [ "text" ] }, "name": "X-Ai/Grok-4-Fast-Reasoning", "open_weights": false, "release_date": "2025-12-18", "schema": 2 }, "xai/grok-4.1-fast": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's fast agentic tool-calling model with a 2M context window; non-reasoning variant for low-latency responses", "family": "grok", "limit": { "context": 2000000, "output": 30000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Grok 4.1 Fast", "open_weights": false, "release_date": "2025-11-19", "schema": 2 }, "xai/grok-4.1-fast-reasoning": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's fast agentic tool-calling model with a 2M context window and built-in reasoning", "family": "grok", "limit": { "context": 2000000, "output": 30000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Grok 4.1 Fast (Reasoning)", "open_weights": false, "release_date": "2025-11-19", "schema": 2 }, "xai/grok-4.20-0309-non-reasoning": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Grok model for agentic tool use, reasoning, coding, and live assistance", "family": "grok", "limit": { "context": 1000000, "output": 30000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "application/pdf" ], "output": [ "text" ] }, "name": "Grok 4.20 (Non-Reasoning)", "open_weights": false, "release_date": "2026-03-09", "schema": 2 }, "xai/grok-4.20-0309-reasoning": { "benchmarks": [ { "dataset": "Original HLE set (2,500 questions)", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 32, "source": "https://artificialanalysis.ai/models/comparisons/grok-4-20-0309-vs-grok-4" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Reasoning Grok for document-heavy analysis and long-horizon tool use", "family": "grok", "limit": { "context": 1000000, "output": 30000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "application/pdf" ], "output": [ "text" ] }, "name": "Grok 4.20 (Reasoning)", "open_weights": false, "release_date": "2026-03-09", "schema": 2 }, "xai/grok-4.3": { "benchmarks": [ { "date": "2026-04-30", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 53, "source": "https://artificialanalysis.ai/articles/xai-launches-grok-4-3-with-improved-agentic-performance-and-lower-pricing", "version": "4.0" }, { "date": "2026-04-30", "metric": "Elo", "name": "GDPval-AA", "score": 1500, "source": "https://artificialanalysis.ai/articles/xai-launches-grok-4-3-with-improved-agentic-performance-and-lower-pricing" }, { "date": "2026-04-30", "metric": "success rate", "name": "τ²-Bench Telecom", "score": 98, "source": "https://artificialanalysis.ai/articles/xai-launches-grok-4-3-with-improved-agentic-performance-and-lower-pricing" }, { "date": "2026-04-30", "metric": "accuracy", "name": "IFBench", "score": 81, "source": "https://artificialanalysis.ai/articles/xai-launches-grok-4-3-with-improved-agentic-performance-and-lower-pricing" }, { "dataset": "Chartography full task set (100 tasks)", "harness": "Inspect AI Chartography harness; leaderboard configuration: 10 runs per task, Gemini 3.5 Flash judge", "name": "Chartography", "score": 12.9, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "High reasoning" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's default Grok for chat, coding, agentic tools, and lower hallucination risk", "family": "grok", "limit": { "context": 1000000, "output": 30000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "application/pdf" ], "output": [ "text" ] }, "name": "Grok 4.3", "open_weights": false, "release_date": "2026-04-17", "schema": 2 }, "xai/grok-4.5": { "benchmarks": [ { "date": "2026-07-08", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 64.7, "source": "https://x.ai/news/grok-4-5" }, { "date": "2026-07-08", "metric": "resolve rate", "name": "SWE-Bench Multilingual", "score": 78, "source": "https://x.ai/news/grok-4-5" }, { "date": "2026-07-08", "metric": "success rate", "name": "Terminal-Bench", "score": 83.3, "source": "https://x.ai/news/grok-4-5", "version": "2.1" }, { "date": "2026-07-08", "metric": "resolve rate", "name": "DeepSWE", "score": 62.0, "source": "https://x.ai/news/grok-4-5", "version": "1.0" }, { "date": "2026-07-08", "harness": "mini-swe-agent", "metric": "resolve rate", "name": "DeepSWE", "score": 53, "source": "https://x.ai/news/grok-4-5", "version": "1.1" }, { "date": "2026-07-08", "metric": "pass@1", "name": "SWE Marathon", "score": 29.0, "source": "https://x.ai/news/grok-4-5" }, { "cost": 2094.11, "cost_source": "https://www.tbench.ai/leaderboard/terminal-bench/4.0", "cost_unit": "USD/evaluation", "dataset": "66 tasks", "effort": "High", "harness": "Grok Build", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 12.4, "source": "https://www.tbench.ai/?efforts=all&version=4.0", "variant": "95% CI: 9.8%–15.0%", "version": "4.0.0" }, { "dataset": "May 2025; text-only; 2,158 questions", "effort": "High", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42.7, "source": "https://modelcap.ai/model/x-ai-grok-4-5", "variant": "no tools; pass@1; reasoning=true" }, { "dataset": "100 tasks", "effort": "Default", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 17.3, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.1 percentage points; frozen launch snapshot July 2026" }, { "dataset": "100 tasks", "effort": "High reasoning", "harness": "Inspect AI; judge=Gemini 3.5 Flash", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 16.7, "source": "https://arxiv.org/abs/2608.10677", "variant": "20 trials per task; no tools; 95% CI ±1.1 percentage points; frozen launch snapshot July 2026" }, { "cost": 2.4157481725663716, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 53.76106194690266, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 5.51917, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 8.586, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's Grok model for chat, coding, agentic tools, and lower hallucination risk", "family": "grok", "limit": { "context": 500000, "output": 500000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Grok 4.5", "open_weights": false, "release_date": "2026-07-08", "schema": 2 }, "xai/grok-4.6": { "benchmarks": [ { "metric": "score", "name": "CursorBench", "score": 40.4, "source": "https://x.ai/news/grok-4-7", "variant": "high effort", "version": "4.0" }, { "dataset": "113 tasks", "effort": "High", "metric": "pass@1", "name": "DeepSWE", "score": 65.2, "source": "https://x.ai/news/grok-4-7", "variant": "high effort", "version": "1.1" }, { "metric": "score", "name": "EEBench", "score": 53, "source": "https://x.ai/news/grok-4-7", "variant": "high effort" }, { "metric": "Elo", "name": "AA-Briefcase", "score": 1546, "source": "https://x.ai/news/grok-4-7", "variant": "high effort", "version": "1.1" }, { "dataset": "66 tasks", "effort": "High", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 20.3, "source": "https://x.ai/news/grok-4-7", "variant": "high effort", "version": "4.0.0" }, { "metric": "score", "name": "Harvey Legal Agent Benchmark", "score": 15.8, "source": "https://x.ai/news/grok-4-7", "variant": "high effort" }, { "metric": "score", "name": "HealthBench", "score": 48.5, "source": "https://x.ai/news/grok-4-7", "variant": "high effort; professional" }, { "metric": "Elo", "name": "GDPval", "score": 1605, "source": "https://x.ai/news/grok-4-7", "variant": "high effort" }, { "cost": 3.4489837522123894, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Medium", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 67.47787610619469, "source": "https://deepswe.datacurve.ai/", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "dataset": "70 tasks across 5 domains", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench-Science", "score": 11.43, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "pass@1; single run; infrastructure/provider-error retries only", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "xHigh", "harness": "Inspect AI; Gemini 3.5 Flash judge", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 20.5, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; leaderboard mean", "version": "unspecified" }, { "dataset": "2,158 text-only questions (Artificial Analysis subset)", "effort": "High", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43, "source": "https://artificialanalysis.ai/models/comparisons/gpt-6-astra-medium-vs-grok-4-6", "variant": "pass@1", "version": "unspecified" }, { "cost": 4.678631982547866, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 31.028795176621, "source": "https://artificialanalysis.ai/models/grok-4-6-xhigh", "version": "5.0.1" }, { "cost": 5.497668745011087, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 66.74057649667405, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 4.3848732239467845, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 65.18847006651885, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 1.0423604231625836, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Low", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 41.648106904231625, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 449 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "cost": 5.062449, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "High", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 17.172, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's frontier model for long-running agents, coding, knowledge work, and visual projects", "family": "grok", "knowledge": "2026-02-01", "limit": { "context": 500000, "output": 500000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "Grok 4.6", "open_weights": false, "release_date": "2026-08-12", "schema": 2 }, "xai/grok-4.7": { "benchmarks": [ { "metric": "score", "name": "CursorBench", "score": 46.3, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort", "version": "4.0" }, { "metric": "score", "name": "DeepSWE", "score": 71, "source": "https://x.ai/news/grok-4-7", "variant": "high effort", "version": "1.1" }, { "metric": "score", "name": "EEBench", "score": 64, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort" }, { "metric": "Elo", "name": "AA-Briefcase", "score": 1657, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort", "version": "1.1" }, { "metric": "score", "name": "Terminal-Bench", "score": 37.6, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort", "version": "4.0.0" }, { "metric": "score", "name": "Harvey Legal Agent Benchmark", "score": 19.6, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort" }, { "metric": "score", "name": "HealthBench", "score": 56.7, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort; professional" }, { "metric": "Elo", "name": "GDPval", "score": 1695, "source": "https://x.ai/news/grok-4-7", "variant": "xhigh effort" }, { "cost": 12.426062537369734, "cost_source": "https://artificialanalysis.ai/models/grok-4-7", "cost_unit": "USD/task", "dataset": "66 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 26, "source": "https://artificialanalysis.ai/models/comparisons/grok-4-7-vs-grok-4-6", "variant": "averaged over 3 repeats", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Xhigh", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 11, "source": "https://artificialanalysis.ai/models/comparisons/grok-4-7-vs-grok-4-6", "variant": "averaged over 3 repeats per task", "version": "0.1.0" }, { "dataset": "100 tasks", "effort": "xHigh", "harness": "Inspect AI", "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 14.7, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "10 runs per task; Gemini 3.5 Flash judge" }, { "cost": 7.771539401587139, "cost_unit": "USD/weighted task", "effort": "Xhigh", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 35.4850117067196, "source": "https://artificialanalysis.ai/models/grok-4-7", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "Xhigh", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1718, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "shell and web access; blind pairwise Elo", "version": "2.1" }, { "dataset": "220 tasks across 44 occupations and 9 industries", "effort": "High", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1716, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "shell and web access; blind pairwise Elo", "version": "2.1" }, { "cost": 11.625468, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "dataset": "GDP-weighted finance, coding, legal, and tax", "effort": "Xhigh", "metric": "performance score", "name": "Vals Index", "score": 58.564, "source": "https://www.vals.ai/models/grok_grok-4.7", "version": "2.2" }, { "cost": 2.9222013011913153, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 31.132077602567, "source": "https://artificialanalysis.ai/models/grok-4-7-low", "version": "5.0.1" }, { "cost": 6.18359556537658, "cost_unit": "USD/weighted task", "effort": "High", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 35.2122962959703, "source": "https://artificialanalysis.ai/models/grok-4-7-high", "version": "5.0.1" }, { "cost": 18.011582, "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Xhigh", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 28.788, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Xhigh", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 43.1, "source": "https://artificialanalysis.ai/models/grok-4-7", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Xhigh", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 23.2, "source": "https://artificialanalysis.ai/models/grok-4-7", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "xAI's frontier model for long-running agents, coding, knowledge work, and visual projects", "family": "grok", "knowledge": "2026-05", "limit": { "context": 500000, "output": 500000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "application/pdf" ], "output": [ "text" ] }, "name": "Grok 4.7", "open_weights": false, "release_date": "2026-09-21", "schema": 2 }, "xai/grok-build-0.1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Grok coding model tuned for agentic engineering and iterative edits", "family": "grok-build", "limit": { "context": 256000, "output": 256000 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "application/pdf" ], "output": [ "text" ] }, "name": "Grok Build 0.1", "open_weights": false, "release_date": "2026-04-16", "schema": 2 }, "xai/grok-code-fast-1": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Fast Grok model for responsive chat, reasoning, and tool-assisted work", "family": "grok", "limit": { "context": 256000, "output": 10000 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "x-AI/Grok-Code-Fast 1", "open_weights": false, "release_date": "2025-09-02", "schema": 2 }, "xai/grok-imagine-image": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "grok", "limit": { "context": 16000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Grok Imagine Image", "open_weights": false, "release_date": "2026-01-28", "schema": 2, "type": "image" }, "xai/grok-imagine-image-2.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Image model for prompt-driven generation, editing, and visual design workflows", "family": "grok", "limit": { "context": 64000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Grok Imagine Image 2.0", "open_weights": false, "release_date": "2026-08-07", "schema": 2, "type": "image" }, "xai/grok-imagine-image-quality": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Higher-fidelity Grok Imagine image model for prompt-driven generation, editing, and visual design workflows", "family": "grok", "limit": { "context": 16000, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "image/jpeg", "image/png" ] }, "name": "Grok Imagine Image Quality", "open_weights": false, "release_date": "2026-04-03", "schema": 2, "type": "image" }, "xai/grok-imagine-video": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text-to-video, image-to-video, video editing and extension with native synchronized audio", "family": "grok", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "video/mp4" ], "output": [ "video/mp4" ] }, "name": "Grok Imagine Video", "open_weights": false, "release_date": "2026-01-28", "schema": 2, "type": "video", "v2_only": true }, "xai/grok-imagine-video-1.5": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video model for text-to-video, image-to-video, and reference-to-video generation with optional voice references", "family": "grok", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp", "audio" ], "output": [ "video/mp4" ] }, "name": "Grok Imagine Video 1.5", "open_weights": false, "release_date": "2026-05-30", "schema": 2, "type": "video" }, "xai/grok-imagine-video-1.5-lite": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Cost-efficient video model for text-to-video and image-to-video generation", "family": "grok", "limit": { "context": 1024, "output": 0 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/webp" ], "output": [ "video/mp4" ] }, "name": "Grok Imagine Video 1.5 Lite", "open_weights": false, "release_date": "2026-10-01", "schema": 2, "type": "video" }, "xai/grok-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Convert text into spoken audio with a single API call. The API supports a rich set of expressive voices, inline speech tags for fine-grained delivery control, and output formats from high-fidelity MP3 to telephony-optimized μ-law.", "limit": { "context": 15000, "output": 15000 }, "modalities": { "input": [ "text" ], "output": [ "audio/mpeg", "audio/wav", "audio/pcm", "audio/basic", "audio/alaw" ] }, "name": "Grok TTS", "open_weights": false, "release_date": "2026-03-16", "schema": 2, "type": "speech" }, "xai/grok-voice-3.0": { "capabilities": { "reasoning": { "supported": true }, "temperature": false, "tools": { "supported": true }, "top_k": false, "top_p": false }, "description": "Speech-to-speech model for real-time voice agents with tool use and optional reasoning", "family": "grok", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "audio/opus" ], "output": [ "text", "audio/pcm", "audio/pcmu", "audio/pcma", "audio/opus" ] }, "name": "Grok Voice 3.0", "open_weights": false, "release_date": "2026-10-14", "schema": 2, "type": "realtime", "v2_only": true }, "xai/grok-voice-transcribe-2.0": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech-to-text model with word-level timestamps, speaker diarization, multichannel audio, and streaming", "family": "grok", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg", "audio/ogg", "audio/flac", "audio/aac", "audio/mp4" ], "output": [ "text" ] }, "name": "Grok Voice Transcribe 2.0", "open_weights": false, "release_date": "2026-09-17", "schema": 2, "type": "transcription" }, "xiaomi/mimo-v2-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "MiMo flash model for fast multimodal assistance and agent workflows", "family": "mimo", "knowledge": "2024-12-01", "last_updated": "2026-02-04", "limit": { "context": 262144, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 15, "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash" ], "total_billion": 309 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiMo-V2-Flash", "open_weights": true, "release_date": "2025-12-16", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash" } ] }, "xiaomi/mimo-v2-omni": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 22.0, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-omni-vs-mimo-v2-pro" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "MiMo omni model for text, image, video, audio, and agents", "family": "mimo", "knowledge": "2024-12", "limit": { "context": 262144, "output": 131072 }, "modalities": { "input": [ "text", "image", "audio", "video", "application/pdf" ], "output": [ "text" ] }, "name": "MiMo-V2-Omni", "open_weights": false, "release_date": "2026-03-18", "schema": 2 }, "xiaomi/mimo-v2-pro": { "benchmarks": [ { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 30, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-pro-vs-gemini-2-5-flash-reasoning" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Earlier MiMo Pro model for multimodal agents, reasoning, and code tasks", "family": "mimo", "knowledge": "2024-12", "limit": { "context": 1048576, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiMo-V2-Pro", "open_weights": false, "release_date": "2026-03-18", "schema": 2 }, "xiaomi/mimo-v2-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "mimo", "limit": { "context": 8192, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "audio" ] }, "name": "MiMo-V2-TTS", "open_weights": false, "release_date": "2026-03-18", "schema": 2, "type": "speech" }, "xiaomi/mimo-v2.5": { "benchmarks": [ { "metric": "score", "name": "MiMo Coding Bench", "score": 71.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "Claw-Eval", "score": 62.3, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "variant": "text" }, { "metric": "score", "name": "Terminal-Bench", "score": 65.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "version": "2.0" }, { "metric": "score", "name": "SWE-Bench Pro", "score": 56.1, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "CharXiv Reasoning", "score": 81, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "MMMU-Pro", "score": 77.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "HR-Bench", "score": 88.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "variant": "4K" }, { "metric": "score", "name": "OmniDocBench", "score": 87.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "Claw-Eval", "score": 23.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5", "variant": "multimodal" }, { "metric": "score", "name": "Video-MME", "score": 87.7, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "DailyOmni", "score": 83.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "metric": "score", "name": "VideoHolmes", "score": 64, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" }, { "dataset": "113 tasks; 3 samples per task", "metric": "task success rate", "name": "DeepSWE", "score": 12.39, "source": "https://arxiv.org/html/2610.40256", "variant": "MiMo-V2.5 w/ Vanilla; 512K context; temperature 1.0; top_p 0.95", "version": "1.1" }, { "dataset": "731 public tasks", "metric": "task success rate", "name": "SWE-Bench Pro", "score": 50.11, "source": "https://arxiv.org/html/2610.40256", "variant": "MiMo-V2.5 w/ Vanilla" }, { "dataset": "Artificial Analysis text-only subset; 2,158 questions", "metric": "pass@1", "name": "Humanity's Last Exam", "score": 27, "source": "https://artificialanalysis.ai/models/comparisons/claude-sonnet-4-6-adaptive-vs-mimo-v2-5-0424", "variant": "MiMo-V2.5 (Artificial Analysis legacy HLE evaluation)" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 996, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-5-pro-vs-mimo-v2-5-0424", "variant": "MiMo-V2.5 (Artificial Analysis comparison)", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open MiMo model for multimodal coding agents and long-context automation", "family": "mimo", "knowledge": "2024-12", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 15, "notes": "Official architecture section's rounded total and active counts; vision and audio encoder details are listed separately. No component sum inferred.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" ], "total_billion": 310 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "audio/mpeg", "audio/wav", "audio/flac", "audio/mp4", "audio/ogg", "video/mp4", "video/quicktime", "video/x-msvideo", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "MiMo-V2.5", "open_weights": true, "release_date": "2026-04-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5" } ] }, "xiaomi/mimo-v2.5-asr": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech recognition model for Chinese, English, dialects, and noisy multi-speaker audio", "family": "mimo", "license": "MIT", "limit": { "context": 8192, "output": 2048 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "ASR checkpoint (without Audio Tokenizer)", "notes": "8,018,587,648 F32 weight/bias values independently counted from all seven pinned shard headers. Config declares untied word embeddings. The inconsistent Hub API total of 7,622,619,136 is not used, and the separate Audio Tokenizer is excluded.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-ASR/blob/227ce4c5e64e356007ad948781f0e8b55a1beba0/model.safetensors.index.json", "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-ASR" ], "total_billion": 8.018587648 } ], "notes": "The ASR checkpoint requires a separately downloaded Audio Tokenizer; its weight count is not a complete pipeline total." } } }, "modalities": { "input": [ "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "MiMo-V2.5-ASR", "open_weights": true, "release_date": "2026-04-23", "schema": 2, "type": "transcription", "v2_only": true, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-ASR" } ] }, "xiaomi/mimo-v2.5-pro": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 78.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro" }, { "date": "2026-04-22", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 57.2, "source": "https://mimo.xiaomi.com/mimo-v2-5-pro/" }, { "date": "2026-04-22", "metric": "accuracy", "name": "GPQA Diamond", "score": 86.6, "source": "https://mimo.xiaomi.com/mimo-v2-5-pro/" }, { "metric": "score", "name": "DeepSWE", "score": 19, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column", "version": "1.1" }, { "metric": "score", "name": "ProgramBench", "score": 12.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "MiMo Code Bench", "score": 40.4, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "AutomationBench", "score": 16, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column", "version": "1.0.6" }, { "metric": "score", "name": "Toolathlon-Verified", "score": 49.1, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "Elo rating", "name": "GDPval-AA", "score": 1107, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column", "version": "2.1" }, { "metric": "score", "name": "Agents' Last Exam", "score": 13.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "Terminal-Bench", "score": 1.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column", "version": "4.0.0" }, { "metric": "score", "name": "Terminal-Bench", "score": 65.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column", "version": "2.1" }, { "metric": "score", "name": "JobBench", "score": 25, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "CyberGym", "score": 40, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "MiMo Cyber Bench", "score": 0, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "ExploitGym", "score": 0.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "ExploitBench", "score": 16.6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "score", "name": "SEC-Bench Pro", "score": 17.7, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "variant": "MiMo-V2.5 Pro comparison column" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 48, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash", "variant": "MiMo-V2.5-Pro-FP8" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 47, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash", "variant": "MiMo-V2.5-Pro-MXFP4" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 34, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash", "variant": "MiMo-V2.5-Pro-FP8; without tool" }, { "metric": "accuracy", "name": "Humanity's Last Exam", "score": 33, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash", "variant": "MiMo-V2.5-Pro-MXFP4; without tool" }, { "dataset": "Artificial Analysis text-only subset; 2,158 questions", "metric": "pass@1", "name": "Humanity's Last Exam", "score": 36, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-5-pro-vs-gpt-5-codex", "variant": "MiMo-V2.5-Pro (Reasoning; Artificial Analysis legacy HLE evaluation)" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1121, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-5-pro-vs-mimo-v2-5-0424", "variant": "MiMo-V2.5-Pro (Reasoning; Artificial Analysis comparison)", "version": "2.1" }, { "dataset": "220 tasks", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1129, "source": "https://artificialanalysis.ai/evaluations/gdpval-aa", "variant": "MiMo-V2.5-Pro (Reasoning; current Artificial Analysis leaderboard)", "version": "2.1" }, { "cost": 0.23303, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks; 3 runs per task (avg@3)", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 0.505, "source": "https://www.vals.ai/models/xiaomi_mimo-v2.5-pro", "variant": "Vals AI; pass@1; temperature 1; top_p 0.95; top_k default; max output 128,000", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution", "family": "mimo", "knowledge": "2024-12", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 42, "notes": "Official model-zoo and architecture table counts, with 1.02T converted to B. Uses the author's rounded architecture convention, not serialized auxiliary tensors.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro" ], "total_billion": 1020 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiMo-V2.5-Pro", "open_weights": true, "release_date": "2026-04-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro" } ] }, "xiaomi/mimo-v2.5-pro-ultraspeed": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "MiMo pro model for strong multimodal reasoning and agent execution", "family": "mimo", "knowledge": "2024-12", "last_updated": "2026-06-09", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 42, "notes": "Official table's target-model counts, with 1.02T converted to B. Excludes the separately listed DFlash draft model; not inferred from FP4 storage.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash" ], "total_billion": 1020 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "MiMo-V2.5-Pro-UltraSpeed", "open_weights": true, "release_date": "2026-06-08", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash" } ] }, "xiaomi/mimo-v2.5-tts": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "mimo", "limit": { "context": 8192, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/pcm" ] }, "name": "MiMo-V2.5-TTS", "open_weights": false, "release_date": "2026-04-22", "schema": 2, "type": "speech" }, "xiaomi/mimo-v2.5-tts-voiceclone": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "mimo", "limit": { "context": 8192, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/pcm" ] }, "name": "MiMo-V2.5-TTS-VoiceClone", "open_weights": false, "release_date": "2026-04-22", "schema": 2, "type": "speech" }, "xiaomi/mimo-v2.5-tts-voicedesign": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech generation model for controllable voice, narration, and audio delivery", "family": "mimo", "limit": { "context": 8192, "output": 8192 }, "modalities": { "input": [ "text" ], "output": [ "audio/wav", "audio/mpeg", "audio/pcm" ] }, "name": "MiMo-V2.5-TTS-VoiceDesign", "open_weights": false, "release_date": "2026-04-22", "schema": 2, "type": "speech" }, "xiaomi/mimo-v2.6-flash": { "benchmarks": [ { "metric": "score", "name": "DeepSWE", "score": 67.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "version": "1.1" }, { "metric": "score", "name": "ProgramBench", "score": 26, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "MiMo Code Bench", "score": 61.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "task_completed_correctly", "name": "AutomationBench", "score": 52.3, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "version": "1.0.6" }, { "metric": "score", "name": "Toolathlon-Verified", "score": 73.6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "Agents' Last Exam", "score": 27.6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 28.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "version": "4.0.0" }, { "metric": "score", "name": "Terminal-Bench", "score": 87.6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL", "version": "2.1" }, { "metric": "score", "name": "OSWorld-Verified", "score": 80.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "JobBench", "score": 61.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "CyberGym", "score": 95.1, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "MiMo Cyber Bench", "score": 77.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "ExploitGym", "score": 6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "ExploitBench", "score": 25.3, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "SEC-Bench Pro", "score": 47.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "metric": "score", "name": "MiMo VisualCoding", "score": 71.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "dataset": "2,158 text-only questions from the original 2,500-item HLE set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 35, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-6-flash-vs-step-5" }, { "cost": 0.221007, "cost_unit": "USD/test", "metric": "performance score", "name": "Vals Index", "score": 54.826, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "variant": "Official Xiaomi API; exact evaluated checkpoint undisclosed; not verified as the linked Flash-RL weights; temperature=1.0; top_p=0.95; max_output_tokens=128000", "version": "2.2" }, { "cost": 0.12852987084014672, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 26.7276981444568, "source": "https://artificialanalysis.ai/models/mimo-v2-6-flash", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1615, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-6-flash-vs-step-5", "version": "2.1" }, { "cost": 0.211804, "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 24.242, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "MiMo Flash model for multimodal coding agents and long-context automation", "family": "mimo", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 15, "notes": "Official architecture section's rounded total and active counts; no draft-model or encoder sum inferred.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" ], "total_billion": 309 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "audio/mpeg", "audio/wav", "audio/flac", "audio/mp4", "audio/ogg", "video/mp4", "video/quicktime", "video/x-msvideo", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "MiMo-V2.6-Flash", "open_weights": true, "release_date": "2026-09-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL" } ] }, "xiaomi/mimo-v2.6-pro": { "benchmarks": [ { "metric": "score", "name": "DeepSWE", "score": 71.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column", "version": "1.1" }, { "metric": "score", "name": "ProgramBench", "score": 26.5, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "MiMo Code Bench", "score": 63.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "task_completed_correctly", "name": "AutomationBench", "score": 53.1, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column", "version": "1.0.6" }, { "metric": "score", "name": "Toolathlon-Verified", "score": 76.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "Elo rating", "name": "GDPval-AA", "score": 1673, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column", "version": "2.1" }, { "metric": "score", "name": "Agents' Last Exam", "score": 31.6, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "task resolution rate", "name": "Terminal-Bench", "score": 34.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column", "version": "4.0.0" }, { "metric": "score", "name": "Terminal-Bench", "score": 89.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column", "version": "2.1" }, { "metric": "score", "name": "OSWorld-Verified", "score": 82, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "JobBench", "score": 62, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "CyberGym", "score": 94, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "MiMo Cyber Bench", "score": 80.2, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "ExploitGym", "score": 17.8, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "ExploitBench", "score": 47.9, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "SEC-Bench Pro", "score": 66.3, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "metric": "score", "name": "MiMo VisualCoding", "score": 72.3, "source": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL", "variant": "MiMo-V2.6 Pro comparison column" }, { "dataset": "2,158 text-only questions from the original 2,500-item HLE set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 49, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-6-pro-vs-gpt-6-astra" }, { "cost": 0.42958, "cost_unit": "USD/test", "metric": "performance score", "name": "Vals Index", "score": 57.813, "source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "variant": "Official Xiaomi API; exact evaluated checkpoint undisclosed; not verified as the linked Pro-RL weights; temperature=1.0; top_p=0.95; max_output_tokens=128000", "version": "2.2" }, { "cost": 0.25718036275876155, "cost_unit": "USD/weighted task", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 29.8897833093321, "source": "https://artificialanalysis.ai/models/mimo-v2-6-pro", "version": "5.0.1" }, { "dataset": "220 tasks across 44 occupations", "harness": "Stirrup", "metric": "Elo rating", "name": "GDPval-AA", "score": 1691, "source": "https://artificialanalysis.ai/models/comparisons/mimo-v2-6-pro-vs-gpt-6-astra", "version": "2.1" }, { "cost": 0.456609, "cost_unit": "USD/test", "dataset": "66 tasks", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 31.313, "source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 15.2, "source": "https://artificialanalysis.ai/models/mimo-v2-6-pro", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution", "family": "mimo", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 42, "notes": "Official architecture section's rounded counts, with 1.02T converted to B; no draft-model or encoder sum inferred.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL" ], "total_billion": 1020 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "audio/mpeg", "audio/wav", "audio/flac", "audio/mp4", "audio/ogg", "video/mp4", "video/quicktime", "video/x-msvideo", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "MiMo-V2.6-Pro", "open_weights": true, "release_date": "2026-09-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL" } ] }, "xiaomi/mimo-v2.6-pro-ultraspeed": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "MiMo pro model for strong multimodal reasoning and agent execution", "family": "mimo", "limit": { "context": 1048576, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 42, "notes": "Target checkpoint linked by this local speed-tier entry. Official rounded architecture counts, with 1.02T converted to B; no draft-model sum inferred.", "sources": [ "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL" ], "total_billion": 1020 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "audio/mpeg", "audio/wav", "audio/flac", "audio/mp4", "audio/ogg", "video/mp4", "video/quicktime", "video/x-msvideo", "video/x-ms-wmv" ], "output": [ "text" ] }, "name": "MiMo-V2.6-Pro-UltraSpeed", "open_weights": true, "release_date": "2026-09-21", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL" } ] }, "zhipuai/autoglm-phone-9b-multilingual": { "capabilities": { "reasoning": { "supported": false }, "temperature": true, "tools": { "supported": false }, "top_k": true, "top_p": true }, "description": "Multilingual visual model for operating smartphone interfaces and mobile agents", "family": "glm", "license": "MIT", "limit": { "context": 65536, "output": 65536 }, "metadata": { "modelatlas": { "parameters": { "notes": "Complete multimodal checkpoint: 10,292,777,472 BF16 weight/bias values independently counted from all five shard headers at the pinned commit. The Hub API's inconsistent total of 934,400 is not used. The official card identifies the GLM-4.1V-9B-Thinking architecture; no count is inferred from its name.", "sources": [ "https://huggingface.co/zai-org/AutoGLM-Phone-9B-Multilingual/blob/832ab5e014b965268dafd944e64cbc8e4bade892/model.safetensors.index.json", "https://huggingface.co/zai-org/AutoGLM-Phone-9B-Multilingual" ], "total_billion": 10.292777472 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png" ], "output": [ "text" ] }, "name": "AutoGLM Phone 9B Multilingual", "open_weights": true, "release_date": "2025-12-10", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/AutoGLM-Phone-9B-Multilingual" } ] }, "zhipuai/cogvideox-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Video generation model for text-to-video, image-to-video, and first/last-frame clips with optional sound effects", "limit": { "context": 512, "output": 0 }, "modalities": { "input": [ "text", "image/png", "image/jpeg" ], "output": [ "video/mp4" ] }, "name": "CogVideoX-3", "open_weights": false, "release_date": "2025-07-15", "schema": 2, "type": "video", "v2_only": true }, "zhipuai/embedding-2": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Zhipu second-generation text embedding model producing 1,024-dimensional vectors", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Embedding-2", "open_weights": false, "release_date": "2023-12", "schema": 2, "type": "embedding", "v2_only": true }, "zhipuai/embedding-3": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Zhipu third-generation text embedding model with 256 to 2,048 dimensions", "limit": { "context": 8192, "output": 0 }, "modalities": { "input": [ "text" ], "output": [ "embedding" ] }, "name": "Embedding-3", "open_weights": false, "release_date": "2024-08", "schema": 2, "type": "embedding", "v2_only": true }, "zhipuai/glm-4.5": { "benchmarks": [ { "date": "2026-03-11", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 26.3, "source": "https://openrouter.ai/z-ai/glm-4.5/benchmarks" }, { "date": "2026-03-11", "metric": "percent correct", "name": "SciCode", "score": 34.8, "source": "https://openrouter.ai/z-ai/glm-4.5/benchmarks" }, { "date": "2026-03-11", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 22, "source": "https://openrouter.ai/z-ai/glm-4.5/benchmarks" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.32, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Hybrid-reasoning GLM release that made the 4.5 line broadly useful", "family": "glm", "knowledge": "2025-04", "limit": { "context": 131072, "output": 98304 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 32, "sources": [ "https://huggingface.co/zai-org/GLM-4.5" ], "total_billion": 355 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.5", "open_weights": true, "release_date": "2025-07-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.5" } ] }, "zhipuai/glm-4.5-air": { "benchmarks": [ { "date": "2026-05-30", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 23.8, "source": "https://openrouter.ai/z-ai/glm-4.5-air/benchmarks" }, { "date": "2026-05-30", "metric": "percent correct", "name": "SciCode", "score": 30.6, "source": "https://openrouter.ai/z-ai/glm-4.5-air/benchmarks" }, { "date": "2026-05-30", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 20.5, "source": "https://openrouter.ai/z-ai/glm-4.5-air/benchmarks" }, { "dataset": "Final original HLE; 2,500 public questions, multimodal", "harness": "Scale Labs leaderboard", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 8.12, "source": "https://labs.scale.com/leaderboard/humanitys_last_exam", "variant": "Scale Labs evaluation; temperature=0.0 when configurable (otherwise provider-stated); o3-mini-2025-01-31 extractor/judge; tool use not stated" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lighter GLM-4.5 variant for fast coding assistance and cheaper agents", "family": "glm-air", "knowledge": "2025-04", "limit": { "context": 131072, "output": 98304 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "sources": [ "https://huggingface.co/zai-org/GLM-4.5-Air" ], "total_billion": 106 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.5-Air", "open_weights": true, "release_date": "2025-07-28", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.5-Air" } ] }, "zhipuai/glm-4.5-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Efficient GLM model for fast reasoning, coding, and agent workflows", "family": "glm-flash", "knowledge": "2025-04", "limit": { "context": 131072, "output": 98304 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.5-Flash", "open_weights": false, "release_date": "2025-07-28", "schema": 2 }, "zhipuai/glm-4.5v": { "benchmarks": [ { "date": "2026-04-29", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 10.9, "source": "https://openrouter.ai/z-ai/glm-4.5v/benchmarks" }, { "date": "2026-04-29", "metric": "percent correct", "name": "SciCode", "score": 22.1, "source": "https://openrouter.ai/z-ai/glm-4.5v/benchmarks" }, { "date": "2026-04-29", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 5.3, "source": "https://openrouter.ai/z-ai/glm-4.5v/benchmarks" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "GLM vision model for visual reasoning, documents, and multimodal agents", "family": "glm", "knowledge": "2025-04", "limit": { "context": 64000, "output": 16384 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 12, "notes": "Total is the Hub-published BF16/F32 count for the complete multimodal checkpoint; active count is explicitly reported by Z.ai. No vision-component sum inferred.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.5V", "https://docs.z.ai/guides/vlm/glm-4.5v.md" ], "total_billion": 107.71093312 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/x-matroska", "video/quicktime" ], "output": [ "text" ] }, "name": "GLM-4.5V", "open_weights": true, "release_date": "2025-08-11", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.5V" } ] }, "zhipuai/glm-4.6": { "benchmarks": [ { "date": "2026-05-22", "metric": "index", "name": "Artificial Analysis Coding Index", "score": 29.5, "source": "https://openrouter.ai/z-ai/glm-4.6/benchmarks" }, { "date": "2026-05-22", "metric": "percent correct", "name": "SciCode", "score": 38.4, "source": "https://openrouter.ai/z-ai/glm-4.6/benchmarks" }, { "date": "2026-05-22", "metric": "success rate", "name": "Terminal-Bench Hard", "score": 25, "source": "https://openrouter.ai/z-ai/glm-4.6/benchmarks" }, { "dataset": "public", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 9.67, "source": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Late GLM-4 workhorse for coding agents, reasoning, and structured tasks", "family": "glm", "knowledge": "2025-04", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the main unquantized checkpoint. The separately shipped auxiliary MTP file is not summed into this reported main-model count.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.6" ], "total_billion": 356.785898816 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.6", "open_weights": true, "release_date": "2025-09-30", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.6" } ] }, "zhipuai/glm-4.6v": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "GLM vision model for visual reasoning, documents, and multimodal agents", "family": "glm", "knowledge": "2025-04", "limit": { "context": 128000, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete multimodal checkpoint, not just the nominal 106B language backbone.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.6V" ], "total_billion": 107.71093312 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/x-matroska", "video/quicktime" ], "output": [ "text" ] }, "name": "GLM-4.6V", "open_weights": true, "release_date": "2025-12-08", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.6V" } ] }, "zhipuai/glm-4.6v-flash": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Lightweight GLM vision model for visual reasoning, documents, and multimodal agents", "family": "glm-flash", "limit": { "context": 128000, "output": 32768 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16 count for the complete multimodal checkpoint, not the nominal 9B backbone designation.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.6V-Flash" ], "total_billion": 10.292777472 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/x-matroska", "video/quicktime" ], "output": [ "text" ] }, "name": "GLM-4.6V-Flash", "open_weights": true, "release_date": "2025-12-08", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.6V-Flash" } ] }, "zhipuai/glm-4.7": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 73.8, "source": "https://huggingface.co/zai-org/GLM-4.7" }, { "metric": "score", "name": "Terminal Bench 2.0", "score": 33.4, "source": "https://huggingface.co/zai-org/GLM-4.7" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Mature GLM model for dependable coding, reasoning, and structured agent tasks", "family": "glm", "knowledge": "2025-04", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the main unquantized checkpoint. The separately shipped auxiliary MTP file is not summed into this reported main-model count.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.7" ], "total_billion": 358.337791296 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.7", "open_weights": true, "release_date": "2025-12-22", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.7" } ] }, "zhipuai/glm-4.7-flash": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 59.2, "source": "https://huggingface.co/zai-org/GLM-4.7-Flash" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Budget GLM lane for fast coding help, routing, and everyday automation", "family": "glm-flash", "knowledge": "2025-04", "limit": { "context": 200000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete checkpoint. No active count inferred from the 30B-A3B designation.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.7-Flash" ], "total_billion": 31.221488576 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.7-Flash", "open_weights": true, "release_date": "2026-01-19", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.7-Flash" } ] }, "zhipuai/glm-4.7-flashx": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Efficient GLM model for fast reasoning, coding, and agent workflows", "family": "glm-flash", "knowledge": "2025-04", "limit": { "context": 200000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "The checkpoint linked by this local speed-tier entry. Hub-published BF16/F32 count; no active count inferred from the size designation.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-4.7-Flash" ], "total_billion": 31.221488576 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-4.7-FlashX", "open_weights": true, "release_date": "2026-01-19", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-4.7-Flash" } ] }, "zhipuai/glm-5": { "benchmarks": [ { "metric": "resolved", "name": "SWE-Bench Verified", "score": 72.8, "source": "https://www.swebench.com/" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Codebase QnA", "score": 20.5, "source": "https://labs.scale.com/leaderboard/sweatlas-qna" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Refactoring", "score": 24.24, "source": "https://labs.scale.com/leaderboard/sweatlas-refactoring" }, { "harness": "Mini-SWE-Agent", "metric": "score", "name": "SWE-Atlas Test Writing", "score": 28.74, "source": "https://labs.scale.com/leaderboard/sweatlas-tw" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "General GLM flagship for coding, analysis, and tool-heavy engineering workflows", "family": "glm", "limit": { "context": 204800, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 40, "sources": [ "https://huggingface.co/zai-org/GLM-5" ], "total_billion": 744 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-5", "open_weights": true, "release_date": "2026-02-12", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-5" } ] }, "zhipuai/glm-5-turbo": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Faster GLM-5 lane for coding agents that need lower latency", "family": "glm", "limit": { "context": 200000, "output": 131072 }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-5-Turbo", "open_weights": false, "release_date": "2026-03-16", "schema": 2 }, "zhipuai/glm-5.1": { "benchmarks": [ { "harness": "Claude Code", "metric": "average pass@1", "name": "Artificial Analysis Coding Agent Index", "score": 52.7, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Atlas Codebase QnA", "score": 73.2, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "dataset": "hard-aa", "harness": "Claude Code", "metric": "pass@1", "name": "SWE-Bench Pro", "score": 19.8, "source": "https://artificialanalysis.ai/agents/coding-agents" }, { "harness": "Claude Code", "metric": "pass@1", "name": "Terminal-Bench", "score": 65.1, "source": "https://artificialanalysis.ai/agents/coding-agents", "version": "2.1" }, { "dataset": "113 tasks", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": "18% ±1%", "source": "https://web.archive.org/web/20260615111157/https://deepswe.datacurve.ai/", "version": "1.1" }, { "dataset": "Terminal-Bench 4.0 (66 tasks)", "harness": "mini-swe-agent; Artificial Analysis evaluation", "metric": "task resolution rate", "name": "Terminal-Bench", "score": "2%", "source": "https://artificialanalysis.ai/models/comparisons/glm-5-1-vs-gpt-5-1-codex-mini", "version": "4.0.0" }, { "name": "Humanity's Last Exam", "score": 31.0, "source": "https://huggingface.co/zai-org/GLM-5.1", "variant": "Z.ai model card; HLE without tools" }, { "name": "Humanity's Last Exam", "score": 52.3, "source": "https://huggingface.co/zai-org/GLM-5.1", "variant": "Z.ai model card; HLE with tools" }, { "dataset": "GDPval-AA v2.1 (220 tasks; 44 occupations)", "harness": "Artificial Analysis Stirrup; E2B sandbox with web fetch, web search, code execution, and file output tools", "metric": "Elo rating", "name": "GDPval-AA", "score": 1103, "source": "https://artificialanalysis.ai/models/comparisons/glm-5-1-vs-gpt-5-1-codex-mini", "variant": "Reasoning", "version": "2.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Strong GLM coding model for agentic engineering, terminals, and repository generation", "family": "glm", "limit": { "context": 200000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for the complete unquantized checkpoint, rather than inheriting the GLM-5 count.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-5.1" ], "total_billion": 753.864139008 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-5.1", "open_weights": true, "release_date": "2026-04-07", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-5.1" } ] }, "zhipuai/glm-5.2": { "benchmarks": [ { "date": "2026-06-16", "metric": "resolve rate", "name": "SWE-Bench Pro", "score": 62.1, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "harness": "Claude Code", "metric": "success rate", "name": "Terminal-Bench", "score": 82.7, "source": "https://z.ai/blog/glm-5.2", "version": "2.1" }, { "date": "2026-06-16", "metric": "dominance", "name": "FrontierSWE", "score": 74.4, "source": "https://z.ai/blog/glm-5.2" }, { "dataset": "text-only subset", "date": "2026-06-16", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 40.5, "source": "https://z.ai/blog/glm-5.2" }, { "dataset": "text-only subset", "date": "2026-06-16", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 54.7, "source": "https://z.ai/blog/glm-5.2", "variant": "with tools" }, { "date": "2026-06-16", "metric": "accuracy", "name": "CritPt", "score": 20.9, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "accuracy", "name": "AIME", "score": 99.2, "source": "https://z.ai/blog/glm-5.2", "version": "2026" }, { "date": "2026-06-16", "metric": "accuracy", "name": "HMMT", "score": 94.4, "source": "https://z.ai/blog/glm-5.2", "version": "November 2025" }, { "date": "2026-06-16", "metric": "accuracy", "name": "HMMT", "score": 92.5, "source": "https://z.ai/blog/glm-5.2", "version": "February 2026" }, { "date": "2026-06-16", "metric": "accuracy", "name": "IMOAnswerBench", "score": 91.0, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "accuracy", "name": "GPQA Diamond", "score": 91.2, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "resolve rate", "name": "NL2Repo", "score": 48.9, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "resolve rate", "name": "DeepSWE", "score": 46.2, "source": "https://z.ai/blog/glm-5.2" }, { "cost": 3.9199075743111114, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 43.77777777777778, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 450 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "date": "2026-06-16", "metric": "score", "name": "Program Bench", "score": 63.7, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "harness": "Terminus 2", "metric": "success rate", "name": "Terminal-Bench", "score": 81.0, "source": "https://z.ai/blog/glm-5.2", "version": "2.1" }, { "date": "2026-06-16", "metric": "score", "name": "PostTrainBench", "score": 34.3, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "resolve rate", "name": "SWE Marathon", "score": 13.0, "source": "https://z.ai/blog/glm-5.2" }, { "dataset": "public subset", "date": "2026-06-16", "metric": "score", "name": "MCP Atlas", "score": 76.8, "source": "https://z.ai/blog/glm-5.2" }, { "date": "2026-06-16", "metric": "score", "name": "Tool-Decathlon", "score": 48.2, "source": "https://z.ai/blog/glm-5.2" }, { "cost": 2.8355164725663715, "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "High", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 36.283185840707965, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 452 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Open flagship GLM for long-horizon coding agents and million-token context work", "family": "glm", "limit": { "context": 1000000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count for this complete unquantized checkpoint, rather than inheriting another GLM-5 version's count.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-5.2" ], "total_billion": 753.32994048 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-5.2", "open_weights": true, "release_date": "2026-06-13", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-5.2" } ] }, "zhipuai/glm-5.3": { "benchmarks": [ { "harness": "Claude Code 2.1.207", "metric": "pass@1", "name": "Terminal-Bench", "score": 88.2, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort", "version": "2.1" }, { "harness": "Claude Code 2.1.207", "metric": "avg@3", "name": "Terminal-Bench", "score": 28.3, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort", "version": "3.0" }, { "effort": "Max", "harness": "mini-swe-agent", "metric": "resolved", "name": "DeepSWE", "score": 66.9, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "temperature=0.95; top_p=1.0; 6h timeout; 400K context", "version": "1.1" }, { "cost": 3.9933584893126386, "cost_source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "cost_unit": "USD/task", "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 68.9578713968958, "source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1", "variant": "4 whole-benchmark passes; 113 tasks attempted; 451 scored attempts; provider/verifier/network errors excluded", "version": "1.1" }, { "metric": "score", "name": "NL2Repo", "score": 58.0, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort" }, { "metric": "almost solved", "name": "ProgramBench", "score": 19.0, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort" }, { "date": "2026-08-14", "metric": "dominance score", "name": "FrontierSWE", "score": 78.1, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort" }, { "harness": "Claude Code 2.1.207", "metric": "score", "name": "SWE-Marathon", "score": 42.5, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; modified anti-cheat checks", "version": "1.1" }, { "harness": "Claude Code 2.1.207", "metric": "weighted average", "name": "PostTrainBench", "score": 39.8, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; 3 runs; modified external-API checks" }, { "harness": "Claude Code 2.1.207", "metric": "pass@1", "name": "CyberGym", "score": 84.5, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort" }, { "dataset": "869 tasks", "harness": "Claude Code 2.1.207", "metric": "tasks solved", "name": "ExploitGym", "score": 105, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; 2h rescaled timeout" }, { "dataset": "869 tasks", "harness": "Claude Code 2.1.207", "metric": "tasks solved", "name": "ExploitGym", "score": 130, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; 6h rescaled timeout" }, { "harness": "Claude Code 2.1.207", "metric": "average coverage", "name": "ExploitBench", "score": 54.4, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; union over 3 revisions" }, { "metric": "pass@1", "name": "Toolathlon-Verified", "score": 73, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort" }, { "metric": "pass@1", "name": "AutomationBench", "score": 48.2, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort", "version": "1.0.6" }, { "harness": "Claude Code 2.1.207", "metric": "score", "name": "Agents' Last Exam", "score": 28.5, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort; CLI" }, { "effort": "Max", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 62.5, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "with tools; temperature=1.0; top_p=0.95; max generation length=163840; max context length=300000; judge=GPT-5.6-Luna (medium)" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1769, "source": "https://huggingface.co/zai-org/GLM-5.3", "variant": "max effort", "version": "2" }, { "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench", "score": 41.9, "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0", "variant": "average over three repeats", "version": "4.0.0" }, { "cost": 9.37436, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "Mini-SWE-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 38.889, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3 (mean of three pass@1 runs); 8-hour task limit; no step or cost limit", "version": "4.0.0" }, { "dataset": "70 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 9.5, "source": "https://artificialanalysis.ai/evaluations/terminal-bench-science", "variant": "average over three repeats", "version": "0.1.0" }, { "harness": "Mini-SWE-agent", "metric": "accuracy", "name": "Terminal-Bench-Science", "score": 5.71, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "version": "0.1.0" }, { "cost": 5.009264601719954, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 31.653381843882, "source": "https://artificialanalysis.ai/models/glm-5-3", "version": "5.0.1" }, { "cost": 8.524625, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 59.729, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 1.5749301149593384, "cost_unit": "USD/weighted task", "effort": "Low", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 23.1079373025379, "source": "https://artificialanalysis.ai/models/glm-5-3-low", "version": "5.0.1" }, { "dataset": "Artificial Analysis HLE text-only subset (2,158 questions from the original 2,500-question set)", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 42.3, "source": "https://artificialanalysis.ai/models/glm-5-3", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" }, { "dataset": "HLE-Diamond; 1,000 questions, including 272 image questions", "effort": "Max", "harness": "Artificial Analysis", "metric": "accuracy", "name": "HLE-Diamond", "score": 17.9, "source": "https://artificialanalysis.ai/models/glm-5-3", "variant": "no tools; pass@1; GPT-5.6 Luna (medium) judge" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Flagship GLM model for long-horizon coding, agents, and complex project delivery", "family": "glm", "limit": { "context": 1000000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "notes": "Hub-published BF16/F32 count from the official full-precision release of GLM-5.3; not inferred from the quantized checkpoint.", "sources": [ "https://huggingface.co/api/models/zai-org/GLM-5.3-BF16", "https://huggingface.co/zai-org/GLM-5.3" ], "total_billion": 753.32994048 } } }, "modalities": { "input": [ "text" ], "output": [ "text" ] }, "name": "GLM-5.3", "open_weights": true, "release_date": "2026-08-14", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-5.3" } ] }, "zhipuai/glm-5.3-flash": { "benchmarks": [ { "harness": "Claude Code 2.1.207", "metric": "pass@1", "name": "Terminal-Bench", "score": 84.3, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort", "version": "2.1" }, { "harness": "mini-swe-agent", "metric": "resolved", "name": "DeepSWE", "score": 63.4, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort", "version": "1.1" }, { "metric": "score", "name": "Agents' Last Exam", "score": 26.3, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort" }, { "metric": "pass@1", "name": "AutomationBench", "score": 48.8, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort", "version": "1.0.6" }, { "dataset": "full set", "metric": "accuracy", "name": "Humanity's Last Exam", "score": 55.3, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort; with tools" }, { "metric": "Elo", "name": "GDPval-AA", "score": 1773, "source": "https://huggingface.co/zai-org/GLM-5.3-Flash", "variant": "max effort", "version": "2" }, { "cost": 1.219907, "cost_source": "https://www.vals.ai/_astro/benchmark_view_terminal_bench_4.g-vQx4Dh.json", "cost_unit": "USD/test", "dataset": "66 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "task resolution rate", "name": "Terminal-Bench", "score": 37.374, "source": "https://www.vals.ai/benchmarks/terminal-bench-4", "variant": "avg@3; each run pass@1", "version": "4.0.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 2.86, "source": "https://www.vals.ai/benchmarks/terminal-bench-science", "variant": "single run", "version": "0.1.0" }, { "dataset": "70 tasks", "harness": "mini-swe-agent", "metric": "pass@1", "name": "Terminal-Bench-Science", "score": 5, "source": "https://artificialanalysis.ai/models/comparisons/glm-5-3-flash-vs-minimax-m3", "variant": "mean over three repeats", "version": "0.1.0" }, { "metric": "accuracy (mean; Pass@1)", "name": "Chartography", "score": 16.3, "source": "https://surgehq.ai/benchmarks/chartography", "variant": "Max reasoning" }, { "cost": 2.048666, "cost_source": "https://www.vals.ai/_astro/benchmark_view_vals_index.CxyItaos.json", "cost_unit": "USD/test", "effort": "Max", "metric": "performance score", "name": "Vals Index", "score": 50.861, "source": "https://www.vals.ai/benchmarks/vals_index", "version": "2.2" }, { "cost": 0.5833103440014082, "cost_unit": "USD/weighted task", "effort": "Max", "metric": "index score", "name": "Artificial Analysis Intelligence Index", "score": 27.803889160883, "source": "https://artificialanalysis.ai/models/glm-5-3-flash", "version": "5.0.1" }, { "dataset": "113 tasks", "effort": "Max", "harness": "mini-swe-agent", "metric": "pass@1", "name": "DeepSWE", "score": 63.39285714285714, "source": "https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json", "variant": "4 whole-benchmark passes; 113 tasks attempted; 448 scored attempts; provider/verifier/network errors excluded", "version": "1.1" } ], "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": true, "top_p": true }, "description": "Native multimodal GLM model for efficient coding and long-horizon agent tasks", "family": "glm-flash", "limit": { "context": 1000000, "output": 131072 }, "metadata": { "modelatlas": { "parameters": { "active_billion": 18, "sources": [ "https://huggingface.co/zai-org/GLM-5.3-Flash" ], "total_billion": 320 } } }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/x-matroska", "video/quicktime", "application/pdf" ], "output": [ "text" ] }, "name": "GLM-5.3-Flash", "open_weights": true, "release_date": "2026-08-26", "schema": 2, "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-5.3-Flash" } ] }, "zhipuai/glm-5v-turbo": { "capabilities": { "reasoning": { "supported": true }, "temperature": true, "tools": { "supported": true }, "top_k": false, "top_p": true }, "description": "Fast GLM vision model for screenshots, documents, and multimodal agent tasks", "family": "glm", "limit": { "context": 200000, "output": 131072 }, "modalities": { "input": [ "text", "image/jpeg", "image/png", "video/mp4", "video/quicktime", "video/x-matroska", "application/pdf" ], "output": [ "text" ] }, "name": "GLM-5V-Turbo", "open_weights": false, "release_date": "2026-04-01", "schema": 2 }, "zhipuai/glm-asr-2512": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "Speech recognition model for multilingual and dialect transcription with custom hotwords", "family": "glm", "limit": { "context": 0, "output": 0 }, "modalities": { "input": [ "audio/wav", "audio/mpeg" ], "output": [ "text" ] }, "name": "GLM-ASR-2512", "open_weights": false, "release_date": "2025-12-10", "schema": 2, "type": "transcription", "v2_only": true }, "zhipuai/glm-image": { "capabilities": { "reasoning": { "supported": false }, "temperature": false, "tools": { "supported": false }, "top_k": false, "top_p": false }, "description": "GLM-Image is an image generation model adopts a hybrid autoregressive + diffusion decoder architecture. In general image generation quality, GLM‑Image aligns with mainstream latent diffusion approaches, but it shows significant advantages in text-rendering and knowledge‑intensive generation scenarios. It performs especially well in tasks requiring precise semantic understanding and complex information expression, while maintaining strong capabilities in high‑fidelity and fine‑grained detail generation. In addition to text‑to‑image generation, GLM‑Image also supports a rich set of image‑to‑image tasks including image editing, style transfer, identity‑preserving generation, and multi‑subject consistency.", "license": "MIT", "limit": { "context": 10240, "output": 0 }, "metadata": { "modelatlas": { "parameters": { "components": [ { "name": "Autoregressive generator", "notes": "Official generator specification with vocabulary expanded for visual tokens; not an estimate from the base model's name.", "sources": [ "https://huggingface.co/zai-org/GLM-Image" ], "total_billion": 9 }, { "name": "Diffusion Decoder", "notes": "Official single-stream DiT decoder specification; other pipeline components are not assumed to be zero.", "sources": [ "https://huggingface.co/zai-org/GLM-Image" ], "total_billion": 7 } ], "notes": "Official component counts are recorded separately; no total is invented for the autoregressive, diffusion and encoding pipeline." } } }, "modalities": { "input": [ "text", "image" ], "output": [ "image" ] }, "name": "GLM-Image", "open_weights": true, "release_date": "2026-01-14", "schema": 2, "type": "image", "weights": [ { "label": "Hugging Face", "url": "https://huggingface.co/zai-org/GLM-Image" } ] } }