{ "generatedAt": "2026-05-28T12:03:41.683Z", "source": "openevals", "title": "OpenEvals leaderboard-data", "dataset": "OpenEvals/leaderboard-data", "config": "default", "split": "train", "totalRows": 105, "fetchedRows": 105, "partial": false, "partialReason": null, "scoreKeys": [ "aime2026_score", "evasionBench_score", "gpqa_score", "gsm8k_score", "hle_score", "hmmt2026_score", "mmluPro_score", "olmOcr_score", "swePro_score", "sweVerified_score", "terminalBench_score", "aggregate_score" ], "topModels": [ { "modelId": "meta-llama-llama-3.1-405b", "modelName": "meta-llama/Llama-3.1-405B", "provider": "meta-llama", "modelType": "open", "parametersBillions": 405.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 96.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 96.8, "aggregate_score": 96.8 } }, { "modelId": "microsoft-phi-3-medium-4k-instruct", "modelName": "microsoft/Phi-3-medium-4k-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 14, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 91, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 91, "aggregate_score": 91 } }, { "modelId": "qwen-qwen2-72b", "modelName": "Qwen/Qwen2-72B", "provider": "Qwen", "modelType": "open", "parametersBillions": 72.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 89.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 89.5, "aggregate_score": 89.5 } }, { "modelId": "microsoft-phi-3.5-mini-instruct", "modelName": "microsoft/Phi-3.5-mini-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 3.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 86.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 86.2, "aggregate_score": 86.2 } }, { "modelId": "internlm-internlm2_5-7b-chat", "modelName": "internlm/internlm2_5-7b-chat", "provider": "internlm", "modelType": "open", "parametersBillions": 7.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 86, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 86, "aggregate_score": 86 } }, { "modelId": "datalab-to-chandra-ocr-2", "modelName": "datalab-to/chandra-ocr-2", "provider": "datalab-to", "modelType": "open", "parametersBillions": 5.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 85.9, "aggregate_score": 85.9 } }, { "modelId": "microsoft-phi-3-mini-4k-instruct", "modelName": "microsoft/Phi-3-mini-4k-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 3.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85.7, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 85.7, "aggregate_score": 85.7 } }, { "modelId": "deepseek-ai-deepseek-r1-0528", "modelName": "deepseek-ai/DeepSeek-R1-0528", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 85, "aggregate_score": 85 } }, { "modelId": "qwen-qwen3-235b-a22b-thinking-2507", "modelName": "Qwen/Qwen3-235B-A22B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 84.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 84.4, "aggregate_score": 84.4 } }, { "modelId": "rednote-hilab-dots.mocr", "modelName": "rednote-hilab/dots.mocr", "provider": "rednote-hilab", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-4bit", "modelName": "mlx-community/dots.mocr-4bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-5bit", "modelName": "mlx-community/dots.mocr-5bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-6bit", "modelName": "mlx-community/dots.mocr-6bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-8bit", "modelName": "mlx-community/dots.mocr-8bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-bf16", "modelName": "mlx-community/dots.mocr-bf16", "provider": "mlx-community", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-mxfp4", "modelName": "mlx-community/dots.mocr-mxfp4", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-mxfp8", "modelName": "mlx-community/dots.mocr-mxfp8", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-nvfp4", "modelName": "mlx-community/dots.mocr-nvfp4", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "lightonai-lightonocr-2-1b", "modelName": "lightonai/LightOnOCR-2-1B", "provider": "lightonai", "modelType": "open", "parametersBillions": 1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.2, "aggregate_score": 83.2 } }, { "modelId": "qwen-qwen3-30b-a3b-thinking-2507", "modelName": "Qwen/Qwen3-30B-A3B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 30.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.15, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "aime2026_score": 87.5, "hmmt2026_score": 78.79, "aggregate_score": 83.15 } }, { "modelId": "datalab-to-chandra", "modelName": "datalab-to/chandra", "provider": "datalab-to", "modelType": "open", "parametersBillions": 8.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.1, "aggregate_score": 83.1 } }, { "modelId": "infly-infinity-parser-7b", "modelName": "infly/Infinity-Parser-7B", "provider": "infly", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.5, "aggregate_score": 82.5 } }, { "modelId": "allenai-olmocr-2-7b-1025-fp8", "modelName": "allenai/olmOCR-2-7B-1025-FP8", "provider": "allenai", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.4, "aggregate_score": 82.4 } }, { "modelId": "titanml-olmocr-2-7b-1025-fp8", "modelName": "TitanML/olmOCR-2-7B-1025-FP8", "provider": "TitanML", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.4, "aggregate_score": 82.4 } }, { "modelId": "qwen-qwen3.5-9b", "modelName": "Qwen/Qwen3.5-9B", "provider": "Qwen", "modelType": "open", "parametersBillions": 9.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 81.98, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "aime2026_score": 92.5, "gpqa_score": 81.7, "hmmt2026_score": 71.21, "mmluPro_score": 82.5, "aggregate_score": 81.98 } }, { "modelId": "deepseek-ai-deepseek-v3-0324", "modelName": "deepseek-ai/DeepSeek-V3-0324", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 81.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 81.2, "aggregate_score": 81.2 } }, { "modelId": "qwen-qwen3-next-80b-a3b-instruct", "modelName": "Qwen/Qwen3-Next-80B-A3B-Instruct", "provider": "Qwen", "modelType": "open", "parametersBillions": 81.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 80.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 80.6, "aggregate_score": 80.6 } }, { "modelId": "paddlepaddle-paddleocr-vl", "modelName": "PaddlePaddle/PaddleOCR-VL", "provider": "PaddlePaddle", "modelType": "open", "parametersBillions": 1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 80, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 80, "aggregate_score": 80 } }, { "modelId": "qwen-qwen2-7b", "modelName": "Qwen/Qwen2-7B", "provider": "Qwen", "modelType": "open", "parametersBillions": 7.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.9, "aggregate_score": 79.9 } }, { "modelId": "baidu-qianfan-ocr", "modelName": "baidu/Qianfan-OCR", "provider": "baidu", "modelType": "open", "parametersBillions": 4.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 79.8, "aggregate_score": 79.8 } }, { "modelId": "internlm-internlm2-chat-20b", "modelName": "internlm/internlm2-chat-20b", "provider": "internlm", "modelType": "open", "parametersBillions": 19.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.6, "aggregate_score": 79.6 } }, { "modelId": "deepseek-ai-deepseek-v2", "modelName": "deepseek-ai/DeepSeek-V2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 235.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.2, "aggregate_score": 79.2 } }, { "modelId": "rednote-hilab-dots.ocr", "modelName": "rednote-hilab/dots.ocr", "provider": "rednote-hilab", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 79.1, "aggregate_score": 79.1 } }, { "modelId": "meituan-longcat-longcat-flash-lite", "modelName": "meituan-longcat/LongCat-Flash-Lite", "provider": "meituan-longcat", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 78.29, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 78.29, "aggregate_score": 78.29 } }, { "modelId": "deepseek-ai-deepseek-r1", "modelName": "deepseek-ai/DeepSeek-R1", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.75, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 71.5, "mmluPro_score": 84, "aggregate_score": 77.75 } }, { "modelId": "jdopensource-joyai-llm-flash", "modelName": "jdopensource/JoyAI-LLM-Flash", "provider": "jdopensource", "modelType": "open", "parametersBillions": 49.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.72, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 74.43, "mmluPro_score": 81.02, "aggregate_score": 77.72 } }, { "modelId": "qwen-qwen3.5-4b", "modelName": "Qwen/Qwen3.5-4B", "provider": "Qwen", "modelType": "open", "parametersBillions": 4.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.65, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 76.2, "mmluPro_score": 79.1, "aggregate_score": 77.65 } }, { "modelId": "deepseek-ai-deepseek-v3", "modelName": "deepseek-ai/DeepSeek-V3", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.85, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gsm8k_score": 89.3, "mmluPro_score": 64.4, "aggregate_score": 76.85 } }, { "modelId": "deepseek-ai-deepseek-ocr-2", "modelName": "deepseek-ai/DeepSeek-OCR-2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 3.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.3, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 76.3, "aggregate_score": 76.3 } }, { "modelId": "lightonai-lightonocr-1b-1025", "modelName": "lightonai/LightOnOCR-1B-1025", "provider": "lightonai", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 76.1, "aggregate_score": 76.1 } }, { "modelId": "deepseek-ai-deepseek-ocr", "modelName": "deepseek-ai/DeepSeek-OCR", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 3.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.7, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.7, "aggregate_score": 75.7 } }, { "modelId": "arcee-ai-trinity-large-preview", "modelName": "arcee-ai/Trinity-Large-Preview", "provider": "arcee-ai", "modelType": "open", "parametersBillions": 398.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "ares-realm-studios-trinity-large-preview-arstest", "modelName": "Ares-Realm-Studios/Trinity-Large-Preview-ARSTEST", "provider": "Ares-Realm-Studios", "modelType": "open", "parametersBillions": 398.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "opendatalab-mineru2.5-2509-1.2b", "modelName": "opendatalab/MinerU2.5-2509-1.2B", "provider": "opendatalab", "modelType": "open", "parametersBillions": 1.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "zai-org-glm-ocr", "modelName": "zai-org/GLM-OCR", "provider": "zai-org", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "multilingual-multimodal-nlp-industrialcoder", "modelName": "Multilingual-Multimodal-NLP/IndustrialCoder", "provider": "Multilingual-Multimodal-NLP", "modelType": "open", "parametersBillions": 32, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 74.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 74.8, "aggregate_score": 74.8 } }, { "modelId": "qwen-qwen3.5-397b-a17b", "modelName": "Qwen/Qwen3.5-397B-A17B", "provider": "Qwen", "modelType": "open", "parametersBillions": 403.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 73.57, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 93.33, "gpqa_score": 88.4, "hle_score": 28.7, "hmmt2026_score": 87.88, "mmluPro_score": 87.8, "sweVerified_score": 76.4, "terminalBench_score": 52.5, "aggregate_score": 73.57 } }, { "modelId": "lm-provers-qed-nano", "modelName": "lm-provers/QED-Nano", "provider": "lm-provers", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 72.69, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "aime2026_score": 82.5, "hmmt2026_score": 62.88, "aggregate_score": 72.69 } }, { "modelId": "moonshotai-kimi-k2.5", "modelName": "moonshotai/Kimi-K2.5", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1058.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.57, "coverageCount": 8, "coveragePercent": 72.7, "scores": { "aime2026_score": 95.83, "gpqa_score": 87.6, "hle_score": 50.2, "hmmt2026_score": 87.12, "mmluPro_score": 87.1, "swePro_score": 50.7, "sweVerified_score": 70.8, "terminalBench_score": 43.2, "aggregate_score": 71.57 } }, { "modelId": "stepfun-ai-step-3.5-flash", "modelName": "stepfun-ai/Step-3.5-Flash", "provider": "stepfun-ai", "modelType": "open", "parametersBillions": 199.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.35, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 96.67, "gpqa_score": 83.5, "hle_score": 23.1, "hmmt2026_score": 86.36, "mmluPro_score": 84.4, "sweVerified_score": 74.4, "terminalBench_score": 51, "aggregate_score": 71.35 } }, { "modelId": "mistralai-mistral-small-4-119b-2603", "modelName": "mistralai/Mistral-Small-4-119B-2603", "provider": "mistralai", "modelType": "open", "parametersBillions": 119.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gpqa_score": 71.2, "aggregate_score": 71.2 } }, { "modelId": "zai-org-glm-5", "modelName": "zai-org/GLM-5", "provider": "zai-org", "modelType": "open", "parametersBillions": 753.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 70.65, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "aime2026_score": 95.83, "gpqa_score": 86, "hle_score": 30.5, "hmmt2026_score": 86.36, "sweVerified_score": 72.8, "terminalBench_score": 52.4, "aggregate_score": 70.65 } }, { "modelId": "fireredteam-firered-ocr", "modelName": "FireRedTeam/FireRed-OCR", "provider": "FireRedTeam", "modelType": "open", "parametersBillions": 2.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 70.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 70.2, "aggregate_score": 70.2 } }, { "modelId": "nanonets-nanonets-ocr2-3b", "modelName": "nanonets/Nanonets-OCR2-3B", "provider": "nanonets", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 69.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 69.5, "aggregate_score": 69.5 } }, { "modelId": "qwen-qwen3.5-27b", "modelName": "Qwen/Qwen3.5-27B", "provider": "Qwen", "modelType": "open", "parametersBillions": 27.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.83, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 90.83, "gpqa_score": 85.5, "hle_score": 24.3, "hmmt2026_score": 81.06, "mmluPro_score": 86.1, "sweVerified_score": 72.4, "terminalBench_score": 41.6, "aggregate_score": 68.83 } }, { "modelId": "qwen-qwen3-4b-thinking-2507", "modelName": "Qwen/Qwen3-4B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.83, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "aime2026_score": 82.5, "gpqa_score": 65.8, "hmmt2026_score": 53.03, "mmluPro_score": 74, "aggregate_score": 68.83 } }, { "modelId": "qwen-qwen3.5-35b-a3b", "modelName": "Qwen/Qwen3.5-35B-A3B", "provider": "Qwen", "modelType": "open", "parametersBillions": 36, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.11, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 93.33, "gpqa_score": 84.2, "hle_score": 22.4, "hmmt2026_score": 81.82, "mmluPro_score": 85.3, "sweVerified_score": 69.2, "terminalBench_score": 40.5, "aggregate_score": 68.11 } }, { "modelId": "moonshotai-kimi-k2-instruct-0905", "modelName": "moonshotai/Kimi-K2-Instruct-0905", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1026.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 66.68, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "evasionBench_score": 66.68, "aggregate_score": 66.68 } }, { "modelId": "gair-openswe-72b", "modelName": "GAIR/OpenSWE-72B", "provider": "GAIR", "modelType": "open", "parametersBillions": 72.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 66, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 66, "aggregate_score": 66 } }, { "modelId": "qwen-qwen3-4b-instruct-2507", "modelName": "Qwen/Qwen3-4B-Instruct-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 65.8, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 62, "mmluPro_score": 69.6, "aggregate_score": 65.8 } }, { "modelId": "deepseek-ai-deepseek-v3.2", "modelName": "deepseek-ai/DeepSeek-V3.2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 685.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64.28, "coverageCount": 9, "coveragePercent": 81.8, "scores": { "aime2026_score": 94.17, "evasionBench_score": 66.88, "gpqa_score": 82.4, "hle_score": 40.8, "hmmt2026_score": 84.09, "mmluPro_score": 85, "swePro_score": 15.56, "sweVerified_score": 70, "terminalBench_score": 39.6, "aggregate_score": 64.28 } }, { "modelId": "zai-org-glm-4.7", "modelName": "zai-org/GLM-4.7", "provider": "zai-org", "modelType": "open", "parametersBillions": 358.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64.15, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "evasionBench_score": 82.91, "gpqa_score": 85.7, "hle_score": 24.8, "mmluPro_score": 84.3, "sweVerified_score": 73.8, "terminalBench_score": 33.4, "aggregate_score": 64.15 } }, { "modelId": "qwen-qwen3.5-122b-a10b", "modelName": "Qwen/Qwen3.5-122B-A10B", "provider": "Qwen", "modelType": "open", "parametersBillions": 125.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 86.6, "hle_score": 25.3, "mmluPro_score": 86.7, "sweVerified_score": 72, "terminalBench_score": 49.4, "aggregate_score": 64 } }, { "modelId": "nvidia-nvidia-nemotron-3-super-120b-a12b-bf16", "modelName": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "provider": "nvidia", "modelType": "open", "parametersBillions": 123.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 62.97, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 90, "gpqa_score": 79.23, "hle_score": 18.26, "hmmt2026_score": 84.85, "mmluPro_score": 83.73, "sweVerified_score": 53.73, "terminalBench_score": 31, "aggregate_score": 62.97 } }, { "modelId": "gair-openswe-32b", "modelName": "GAIR/OpenSWE-32B", "provider": "GAIR", "modelType": "open", "parametersBillions": 32.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 62.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 62.4, "aggregate_score": 62.4 } }, { "modelId": "moonshotai-kimi-k2-thinking", "modelName": "moonshotai/Kimi-K2-Thinking", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1058.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 60, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 84.5, "hle_score": 23.9, "mmluPro_score": 84.6, "sweVerified_score": 71.3, "terminalBench_score": 35.7, "aggregate_score": 60 } }, { "modelId": "minimaxai-minimax-m2.5", "modelName": "MiniMaxAI/MiniMax-M2.5", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 58.95, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "gpqa_score": 85.2, "hle_score": 19.4, "swePro_score": 55.4, "sweVerified_score": 75.8, "aggregate_score": 58.95 } }, { "modelId": "lgai-exaone-k-exaone-236b-a23b", "modelName": "LGAI-EXAONE/K-EXAONE-236B-A23B", "provider": "LGAI-EXAONE", "modelType": "open", "parametersBillions": 237.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 58.83, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 79.1, "hle_score": 13.6, "mmluPro_score": 83.8, "aggregate_score": 58.83 } }, { "modelId": "nvidia-nemotron-cascade-2-30b-a3b", "modelName": "nvidia/Nemotron-Cascade-2-30B-A3B", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 57.87, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 76.1, "hle_score": 17.7, "mmluPro_score": 79.8, "aggregate_score": 57.87 } }, { "modelId": "qwen-qwen3.5-2b", "modelName": "Qwen/Qwen3.5-2B", "provider": "Qwen", "modelType": "open", "parametersBillions": 2.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 55.3, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 55.3, "aggregate_score": 55.3 } }, { "modelId": "meta-llama-llama-3.1-8b-instruct", "modelName": "meta-llama/Llama-3.1-8B-Instruct", "provider": "meta-llama", "modelType": "open", "parametersBillions": 8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 54.4, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 30.4, "gsm8k_score": 84.5, "mmluPro_score": 48.3, "aggregate_score": 54.4 } }, { "modelId": "facebook-cwm", "modelName": "facebook/cwm", "provider": "facebook", "modelType": "open", "parametersBillions": 32.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 53.9, "aggregate_score": 53.9 } }, { "modelId": "minimaxai-minimax-m2.1", "modelName": "MiniMaxAI/MiniMax-M2.1", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.59, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "evasionBench_score": 71.31, "hle_score": 22.2, "mmluPro_score": 88, "swePro_score": 36.81, "sweVerified_score": 74, "terminalBench_score": 29.2, "aggregate_score": 53.59 } }, { "modelId": "redhatai-nvidia-nemotron-3-super-120b-a12b-bf16", "modelName": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "provider": "RedHatAI", "modelType": "open", "parametersBillions": 123.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.19, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 79.23, "hle_score": 18.26, "mmluPro_score": 83.73, "sweVerified_score": 53.73, "terminalBench_score": 31, "aggregate_score": 53.19 } }, { "modelId": "meituan-longcat-longcat-flash-thinking-2601", "modelName": "meituan-longcat/LongCat-Flash-Thinking-2601", "provider": "meituan-longcat", "modelType": "open", "parametersBillions": 561.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 52.85, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 80.5, "hle_score": 25.2, "aggregate_score": 52.85 } }, { "modelId": "swe-lego-swe-lego-qwen3-32b", "modelName": "SWE-Lego/SWE-Lego-Qwen3-32B", "provider": "SWE-Lego", "modelType": "open", "parametersBillions": 0, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 52.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 52.6, "aggregate_score": 52.6 } }, { "modelId": "snurfyai-sn-logicer-0.8b", "modelName": "SnurfyAI/Sn-Logicer-0.8B", "provider": "SnurfyAI", "modelType": "open", "parametersBillions": 0.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 50.57, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 50.57, "aggregate_score": 50.57 } }, { "modelId": "qwen-qwen3-coder-next", "modelName": "Qwen/Qwen3-Coder-Next", "provider": "Qwen", "modelType": "open", "parametersBillions": 79.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 50.37, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "swePro_score": 44.3, "sweVerified_score": 70.6, "terminalBench_score": 36.2, "aggregate_score": 50.37 } }, { "modelId": "zai-org-glm-4.7-flash", "modelName": "zai-org/GLM-4.7-Flash", "provider": "zai-org", "modelType": "open", "parametersBillions": 31.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 49.6, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 75.2, "hle_score": 14.4, "sweVerified_score": 59.2, "aggregate_score": 49.6 } }, { "modelId": "minimaxai-minimax-m2", "modelName": "MiniMaxAI/MiniMax-M2", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 48.48, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "hle_score": 12.5, "mmluPro_score": 82, "sweVerified_score": 69.4, "terminalBench_score": 30, "aggregate_score": 48.48 } }, { "modelId": "nanbeige-nanbeige4.1-3b", "modelName": "Nanbeige/Nanbeige4.1-3B", "provider": "Nanbeige", "modelType": "open", "parametersBillions": 3.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 48.2, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 83.8, "hle_score": 12.6, "aggregate_score": 48.2 } }, { "modelId": "qwen-qwen3-coder-480b-a35b-instruct", "modelName": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "provider": "Qwen", "modelType": "open", "parametersBillions": 480.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 46.92, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "evasionBench_score": 78.16, "swePro_score": 38.7, "terminalBench_score": 23.9, "aggregate_score": 46.92 } }, { "modelId": "nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16", "modelName": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 46.9, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 15.5, "mmluPro_score": 78.3, "aggregate_score": 46.9 } }, { "modelId": "nvidia-nvidia-nemotron-3-nano-30b-a3b-fp8", "modelName": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 44.15, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 10.2, "mmluPro_score": 78.1, "aggregate_score": 44.15 } }, { "modelId": "swe-lego-swe-lego-qwen3-8b", "modelName": "SWE-Lego/SWE-Lego-Qwen3-8B", "provider": "SWE-Lego", "modelType": "open", "parametersBillions": 0, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 42.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 42.2, "aggregate_score": 42.2 } }, { "modelId": "tiiuae-falcon-h1r-7b", "modelName": "tiiuae/Falcon-H1R-7B", "provider": "tiiuae", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 41.6, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 11.1, "mmluPro_score": 72.1, "aggregate_score": 41.6 } }, { "modelId": "miromind-ai-mirothinker-v1.5-235b", "modelName": "miromind-ai/MiroThinker-v1.5-235B", "provider": "miromind-ai", "modelType": "open", "parametersBillions": 235.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 39.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 39.2, "aggregate_score": 39.2 } }, { "modelId": "liquidai-lfm2.5-1.2b-instruct", "modelName": "LiquidAI/LFM2.5-1.2B-Instruct", "provider": "LiquidAI", "modelType": "open", "parametersBillions": 1.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 38.89, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gpqa_score": 38.89, "aggregate_score": 38.89 } }, { "modelId": "nvidia-nemotron-orchestrator-8b", "modelName": "nvidia/Nemotron-Orchestrator-8B", "provider": "nvidia", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 37.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 37.1, "aggregate_score": 37.1 } }, { "modelId": "openai-gpt-oss-120b", "modelName": "openai/gpt-oss-120b", "provider": "openai", "modelType": "open", "parametersBillions": 120.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 34.1, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "gpqa_score": 67.1, "hle_score": 5.2, "swePro_score": 16.2, "sweVerified_score": 47.9, "aggregate_score": 34.1 } }, { "modelId": "openai-gpt-oss-20b", "modelName": "openai/gpt-oss-20b", "provider": "openai", "modelType": "open", "parametersBillions": 21.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 32.8, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 56.8, "hle_score": 4.2, "sweVerified_score": 37.4, "aggregate_score": 32.8 } }, { "modelId": "miromind-ai-mirothinker-v1.5-30b", "modelName": "miromind-ai/MiroThinker-v1.5-30B", "provider": "miromind-ai", "modelType": "open", "parametersBillions": 30.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 31, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 31, "aggregate_score": 31 } }, { "modelId": "moonshotai-kimi-k2-instruct", "modelName": "moonshotai/Kimi-K2-Instruct", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1026.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 27.73, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "swePro_score": 27.67, "terminalBench_score": 27.8, "aggregate_score": 27.73 } }, { "modelId": "nvidia-nemotron-terminal-32b", "modelName": "nvidia/Nemotron-Terminal-32B", "provider": "nvidia", "modelType": "open", "parametersBillions": 32.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 27.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "terminalBench_score": 27.4, "aggregate_score": 27.4 } }, { "modelId": "xiaomimimo-mimo-v2-flash", "modelName": "XiaomiMiMo/MiMo-V2-Flash", "provider": "XiaomiMiMo", "modelType": "open", "parametersBillions": 309.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 22.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 22.1, "aggregate_score": 22.1 } }, { "modelId": "qwen-qwen3-235b-a22b", "modelName": "Qwen/Qwen3-235B-A22B", "provider": "Qwen", "modelType": "open", "parametersBillions": 235.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 21.41, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "swePro_score": 21.41, "aggregate_score": 21.41 } }, { "modelId": "qwen-qwen3.5-0.8b", "modelName": "Qwen/Qwen3.5-0.8B", "provider": "Qwen", "modelType": "open", "parametersBillions": 0.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 20.8, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 11.9, "mmluPro_score": 29.7, "aggregate_score": 20.8 } }, { "modelId": "nvidia-nemotron-terminal-14b", "modelName": "nvidia/Nemotron-Terminal-14B", "provider": "nvidia", "modelType": "open", "parametersBillions": 14.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 20.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "terminalBench_score": 20.2, "aggregate_score": 20.2 } }, { "modelId": "openbmb-agentcpm-explore", "modelName": "openbmb/AgentCPM-Explore", "provider": "openbmb", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 19.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 19.1, "aggregate_score": 19.1 } }, { "modelId": "zai-org-glm-4.6", "modelName": "zai-org/GLM-4.6", "provider": "zai-org", "modelType": "open", "parametersBillions": 356.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 17.09, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "swePro_score": 9.67, "terminalBench_score": 24.5, "aggregate_score": 17.09 } } ], "models": [ { "modelId": "meta-llama-llama-3.1-405b", "modelName": "meta-llama/Llama-3.1-405B", "provider": "meta-llama", "modelType": "open", "parametersBillions": 405.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 96.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 96.8, "aggregate_score": 96.8 } }, { "modelId": "microsoft-phi-3-medium-4k-instruct", "modelName": "microsoft/Phi-3-medium-4k-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 14, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 91, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 91, "aggregate_score": 91 } }, { "modelId": "qwen-qwen2-72b", "modelName": "Qwen/Qwen2-72B", "provider": "Qwen", "modelType": "open", "parametersBillions": 72.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 89.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 89.5, "aggregate_score": 89.5 } }, { "modelId": "microsoft-phi-3.5-mini-instruct", "modelName": "microsoft/Phi-3.5-mini-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 3.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 86.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 86.2, "aggregate_score": 86.2 } }, { "modelId": "internlm-internlm2_5-7b-chat", "modelName": "internlm/internlm2_5-7b-chat", "provider": "internlm", "modelType": "open", "parametersBillions": 7.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 86, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 86, "aggregate_score": 86 } }, { "modelId": "datalab-to-chandra-ocr-2", "modelName": "datalab-to/chandra-ocr-2", "provider": "datalab-to", "modelType": "open", "parametersBillions": 5.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 85.9, "aggregate_score": 85.9 } }, { "modelId": "microsoft-phi-3-mini-4k-instruct", "modelName": "microsoft/Phi-3-mini-4k-instruct", "provider": "microsoft", "modelType": "open", "parametersBillions": 3.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85.7, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 85.7, "aggregate_score": 85.7 } }, { "modelId": "deepseek-ai-deepseek-r1-0528", "modelName": "deepseek-ai/DeepSeek-R1-0528", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 85, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 85, "aggregate_score": 85 } }, { "modelId": "qwen-qwen3-235b-a22b-thinking-2507", "modelName": "Qwen/Qwen3-235B-A22B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 84.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 84.4, "aggregate_score": 84.4 } }, { "modelId": "rednote-hilab-dots.mocr", "modelName": "rednote-hilab/dots.mocr", "provider": "rednote-hilab", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-4bit", "modelName": "mlx-community/dots.mocr-4bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-5bit", "modelName": "mlx-community/dots.mocr-5bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-6bit", "modelName": "mlx-community/dots.mocr-6bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-8bit", "modelName": "mlx-community/dots.mocr-8bit", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-bf16", "modelName": "mlx-community/dots.mocr-bf16", "provider": "mlx-community", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-mxfp4", "modelName": "mlx-community/dots.mocr-mxfp4", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-mxfp8", "modelName": "mlx-community/dots.mocr-mxfp8", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "mlx-community-dots.mocr-nvfp4", "modelName": "mlx-community/dots.mocr-nvfp4", "provider": "mlx-community", "modelType": "open", "parametersBillions": 1.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.9, "aggregate_score": 83.9 } }, { "modelId": "lightonai-lightonocr-2-1b", "modelName": "lightonai/LightOnOCR-2-1B", "provider": "lightonai", "modelType": "open", "parametersBillions": 1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.2, "aggregate_score": 83.2 } }, { "modelId": "qwen-qwen3-30b-a3b-thinking-2507", "modelName": "Qwen/Qwen3-30B-A3B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 30.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.15, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "aime2026_score": 87.5, "hmmt2026_score": 78.79, "aggregate_score": 83.15 } }, { "modelId": "datalab-to-chandra", "modelName": "datalab-to/chandra", "provider": "datalab-to", "modelType": "open", "parametersBillions": 8.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 83.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 83.1, "aggregate_score": 83.1 } }, { "modelId": "infly-infinity-parser-7b", "modelName": "infly/Infinity-Parser-7B", "provider": "infly", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.5, "aggregate_score": 82.5 } }, { "modelId": "allenai-olmocr-2-7b-1025-fp8", "modelName": "allenai/olmOCR-2-7B-1025-FP8", "provider": "allenai", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.4, "aggregate_score": 82.4 } }, { "modelId": "titanml-olmocr-2-7b-1025-fp8", "modelName": "TitanML/olmOCR-2-7B-1025-FP8", "provider": "TitanML", "modelType": "open", "parametersBillions": 8.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 82.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 82.4, "aggregate_score": 82.4 } }, { "modelId": "qwen-qwen3.5-9b", "modelName": "Qwen/Qwen3.5-9B", "provider": "Qwen", "modelType": "open", "parametersBillions": 9.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 81.98, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "aime2026_score": 92.5, "gpqa_score": 81.7, "hmmt2026_score": 71.21, "mmluPro_score": 82.5, "aggregate_score": 81.98 } }, { "modelId": "deepseek-ai-deepseek-v3-0324", "modelName": "deepseek-ai/DeepSeek-V3-0324", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 81.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 81.2, "aggregate_score": 81.2 } }, { "modelId": "qwen-qwen3-next-80b-a3b-instruct", "modelName": "Qwen/Qwen3-Next-80B-A3B-Instruct", "provider": "Qwen", "modelType": "open", "parametersBillions": 81.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 80.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 80.6, "aggregate_score": 80.6 } }, { "modelId": "paddlepaddle-paddleocr-vl", "modelName": "PaddlePaddle/PaddleOCR-VL", "provider": "PaddlePaddle", "modelType": "open", "parametersBillions": 1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 80, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 80, "aggregate_score": 80 } }, { "modelId": "qwen-qwen2-7b", "modelName": "Qwen/Qwen2-7B", "provider": "Qwen", "modelType": "open", "parametersBillions": 7.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.9, "aggregate_score": 79.9 } }, { "modelId": "baidu-qianfan-ocr", "modelName": "baidu/Qianfan-OCR", "provider": "baidu", "modelType": "open", "parametersBillions": 4.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 79.8, "aggregate_score": 79.8 } }, { "modelId": "internlm-internlm2-chat-20b", "modelName": "internlm/internlm2-chat-20b", "provider": "internlm", "modelType": "open", "parametersBillions": 19.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.6, "aggregate_score": 79.6 } }, { "modelId": "deepseek-ai-deepseek-v2", "modelName": "deepseek-ai/DeepSeek-V2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 235.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 79.2, "aggregate_score": 79.2 } }, { "modelId": "rednote-hilab-dots.ocr", "modelName": "rednote-hilab/dots.ocr", "provider": "rednote-hilab", "modelType": "open", "parametersBillions": 3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 79.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 79.1, "aggregate_score": 79.1 } }, { "modelId": "meituan-longcat-longcat-flash-lite", "modelName": "meituan-longcat/LongCat-Flash-Lite", "provider": "meituan-longcat", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 78.29, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 78.29, "aggregate_score": 78.29 } }, { "modelId": "deepseek-ai-deepseek-r1", "modelName": "deepseek-ai/DeepSeek-R1", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.75, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 71.5, "mmluPro_score": 84, "aggregate_score": 77.75 } }, { "modelId": "jdopensource-joyai-llm-flash", "modelName": "jdopensource/JoyAI-LLM-Flash", "provider": "jdopensource", "modelType": "open", "parametersBillions": 49.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.72, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 74.43, "mmluPro_score": 81.02, "aggregate_score": 77.72 } }, { "modelId": "qwen-qwen3.5-4b", "modelName": "Qwen/Qwen3.5-4B", "provider": "Qwen", "modelType": "open", "parametersBillions": 4.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 77.65, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 76.2, "mmluPro_score": 79.1, "aggregate_score": 77.65 } }, { "modelId": "deepseek-ai-deepseek-v3", "modelName": "deepseek-ai/DeepSeek-V3", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 684.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.85, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gsm8k_score": 89.3, "mmluPro_score": 64.4, "aggregate_score": 76.85 } }, { "modelId": "deepseek-ai-deepseek-ocr-2", "modelName": "deepseek-ai/DeepSeek-OCR-2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 3.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.3, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 76.3, "aggregate_score": 76.3 } }, { "modelId": "lightonai-lightonocr-1b-1025", "modelName": "lightonai/LightOnOCR-1B-1025", "provider": "lightonai", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 76.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 76.1, "aggregate_score": 76.1 } }, { "modelId": "deepseek-ai-deepseek-ocr", "modelName": "deepseek-ai/DeepSeek-OCR", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 3.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.7, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.7, "aggregate_score": 75.7 } }, { "modelId": "arcee-ai-trinity-large-preview", "modelName": "arcee-ai/Trinity-Large-Preview", "provider": "arcee-ai", "modelType": "open", "parametersBillions": 398.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "ares-realm-studios-trinity-large-preview-arstest", "modelName": "Ares-Realm-Studios/Trinity-Large-Preview-ARSTEST", "provider": "Ares-Realm-Studios", "modelType": "open", "parametersBillions": 398.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "opendatalab-mineru2.5-2509-1.2b", "modelName": "opendatalab/MinerU2.5-2509-1.2B", "provider": "opendatalab", "modelType": "open", "parametersBillions": 1.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "zai-org-glm-ocr", "modelName": "zai-org/GLM-OCR", "provider": "zai-org", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 75.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 75.2, "aggregate_score": 75.2 } }, { "modelId": "multilingual-multimodal-nlp-industrialcoder", "modelName": "Multilingual-Multimodal-NLP/IndustrialCoder", "provider": "Multilingual-Multimodal-NLP", "modelType": "open", "parametersBillions": 32, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 74.8, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 74.8, "aggregate_score": 74.8 } }, { "modelId": "qwen-qwen3.5-397b-a17b", "modelName": "Qwen/Qwen3.5-397B-A17B", "provider": "Qwen", "modelType": "open", "parametersBillions": 403.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 73.57, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 93.33, "gpqa_score": 88.4, "hle_score": 28.7, "hmmt2026_score": 87.88, "mmluPro_score": 87.8, "sweVerified_score": 76.4, "terminalBench_score": 52.5, "aggregate_score": 73.57 } }, { "modelId": "lm-provers-qed-nano", "modelName": "lm-provers/QED-Nano", "provider": "lm-provers", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 72.69, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "aime2026_score": 82.5, "hmmt2026_score": 62.88, "aggregate_score": 72.69 } }, { "modelId": "moonshotai-kimi-k2.5", "modelName": "moonshotai/Kimi-K2.5", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1058.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.57, "coverageCount": 8, "coveragePercent": 72.7, "scores": { "aime2026_score": 95.83, "gpqa_score": 87.6, "hle_score": 50.2, "hmmt2026_score": 87.12, "mmluPro_score": 87.1, "swePro_score": 50.7, "sweVerified_score": 70.8, "terminalBench_score": 43.2, "aggregate_score": 71.57 } }, { "modelId": "stepfun-ai-step-3.5-flash", "modelName": "stepfun-ai/Step-3.5-Flash", "provider": "stepfun-ai", "modelType": "open", "parametersBillions": 199.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.35, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 96.67, "gpqa_score": 83.5, "hle_score": 23.1, "hmmt2026_score": 86.36, "mmluPro_score": 84.4, "sweVerified_score": 74.4, "terminalBench_score": 51, "aggregate_score": 71.35 } }, { "modelId": "mistralai-mistral-small-4-119b-2603", "modelName": "mistralai/Mistral-Small-4-119B-2603", "provider": "mistralai", "modelType": "open", "parametersBillions": 119.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 71.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gpqa_score": 71.2, "aggregate_score": 71.2 } }, { "modelId": "zai-org-glm-5", "modelName": "zai-org/GLM-5", "provider": "zai-org", "modelType": "open", "parametersBillions": 753.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 70.65, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "aime2026_score": 95.83, "gpqa_score": 86, "hle_score": 30.5, "hmmt2026_score": 86.36, "sweVerified_score": 72.8, "terminalBench_score": 52.4, "aggregate_score": 70.65 } }, { "modelId": "fireredteam-firered-ocr", "modelName": "FireRedTeam/FireRed-OCR", "provider": "FireRedTeam", "modelType": "open", "parametersBillions": 2.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 70.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 70.2, "aggregate_score": 70.2 } }, { "modelId": "nanonets-nanonets-ocr2-3b", "modelName": "nanonets/Nanonets-OCR2-3B", "provider": "nanonets", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 69.5, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "olmOcr_score": 69.5, "aggregate_score": 69.5 } }, { "modelId": "qwen-qwen3.5-27b", "modelName": "Qwen/Qwen3.5-27B", "provider": "Qwen", "modelType": "open", "parametersBillions": 27.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.83, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 90.83, "gpqa_score": 85.5, "hle_score": 24.3, "hmmt2026_score": 81.06, "mmluPro_score": 86.1, "sweVerified_score": 72.4, "terminalBench_score": 41.6, "aggregate_score": 68.83 } }, { "modelId": "qwen-qwen3-4b-thinking-2507", "modelName": "Qwen/Qwen3-4B-Thinking-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.83, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "aime2026_score": 82.5, "gpqa_score": 65.8, "hmmt2026_score": 53.03, "mmluPro_score": 74, "aggregate_score": 68.83 } }, { "modelId": "qwen-qwen3.5-35b-a3b", "modelName": "Qwen/Qwen3.5-35B-A3B", "provider": "Qwen", "modelType": "open", "parametersBillions": 36, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 68.11, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 93.33, "gpqa_score": 84.2, "hle_score": 22.4, "hmmt2026_score": 81.82, "mmluPro_score": 85.3, "sweVerified_score": 69.2, "terminalBench_score": 40.5, "aggregate_score": 68.11 } }, { "modelId": "moonshotai-kimi-k2-instruct-0905", "modelName": "moonshotai/Kimi-K2-Instruct-0905", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1026.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 66.68, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "evasionBench_score": 66.68, "aggregate_score": 66.68 } }, { "modelId": "gair-openswe-72b", "modelName": "GAIR/OpenSWE-72B", "provider": "GAIR", "modelType": "open", "parametersBillions": 72.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 66, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 66, "aggregate_score": 66 } }, { "modelId": "qwen-qwen3-4b-instruct-2507", "modelName": "Qwen/Qwen3-4B-Instruct-2507", "provider": "Qwen", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 65.8, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 62, "mmluPro_score": 69.6, "aggregate_score": 65.8 } }, { "modelId": "deepseek-ai-deepseek-v3.2", "modelName": "deepseek-ai/DeepSeek-V3.2", "provider": "deepseek-ai", "modelType": "open", "parametersBillions": 685.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64.28, "coverageCount": 9, "coveragePercent": 81.8, "scores": { "aime2026_score": 94.17, "evasionBench_score": 66.88, "gpqa_score": 82.4, "hle_score": 40.8, "hmmt2026_score": 84.09, "mmluPro_score": 85, "swePro_score": 15.56, "sweVerified_score": 70, "terminalBench_score": 39.6, "aggregate_score": 64.28 } }, { "modelId": "zai-org-glm-4.7", "modelName": "zai-org/GLM-4.7", "provider": "zai-org", "modelType": "open", "parametersBillions": 358.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64.15, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "evasionBench_score": 82.91, "gpqa_score": 85.7, "hle_score": 24.8, "mmluPro_score": 84.3, "sweVerified_score": 73.8, "terminalBench_score": 33.4, "aggregate_score": 64.15 } }, { "modelId": "qwen-qwen3.5-122b-a10b", "modelName": "Qwen/Qwen3.5-122B-A10B", "provider": "Qwen", "modelType": "open", "parametersBillions": 125.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 64, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 86.6, "hle_score": 25.3, "mmluPro_score": 86.7, "sweVerified_score": 72, "terminalBench_score": 49.4, "aggregate_score": 64 } }, { "modelId": "nvidia-nvidia-nemotron-3-super-120b-a12b-bf16", "modelName": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "provider": "nvidia", "modelType": "open", "parametersBillions": 123.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 62.97, "coverageCount": 7, "coveragePercent": 63.6, "scores": { "aime2026_score": 90, "gpqa_score": 79.23, "hle_score": 18.26, "hmmt2026_score": 84.85, "mmluPro_score": 83.73, "sweVerified_score": 53.73, "terminalBench_score": 31, "aggregate_score": 62.97 } }, { "modelId": "gair-openswe-32b", "modelName": "GAIR/OpenSWE-32B", "provider": "GAIR", "modelType": "open", "parametersBillions": 32.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 62.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 62.4, "aggregate_score": 62.4 } }, { "modelId": "moonshotai-kimi-k2-thinking", "modelName": "moonshotai/Kimi-K2-Thinking", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1058.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 60, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 84.5, "hle_score": 23.9, "mmluPro_score": 84.6, "sweVerified_score": 71.3, "terminalBench_score": 35.7, "aggregate_score": 60 } }, { "modelId": "minimaxai-minimax-m2.5", "modelName": "MiniMaxAI/MiniMax-M2.5", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 58.95, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "gpqa_score": 85.2, "hle_score": 19.4, "swePro_score": 55.4, "sweVerified_score": 75.8, "aggregate_score": 58.95 } }, { "modelId": "lgai-exaone-k-exaone-236b-a23b", "modelName": "LGAI-EXAONE/K-EXAONE-236B-A23B", "provider": "LGAI-EXAONE", "modelType": "open", "parametersBillions": 237.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 58.83, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 79.1, "hle_score": 13.6, "mmluPro_score": 83.8, "aggregate_score": 58.83 } }, { "modelId": "nvidia-nemotron-cascade-2-30b-a3b", "modelName": "nvidia/Nemotron-Cascade-2-30B-A3B", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 57.87, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 76.1, "hle_score": 17.7, "mmluPro_score": 79.8, "aggregate_score": 57.87 } }, { "modelId": "qwen-qwen3.5-2b", "modelName": "Qwen/Qwen3.5-2B", "provider": "Qwen", "modelType": "open", "parametersBillions": 2.3, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 55.3, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "mmluPro_score": 55.3, "aggregate_score": 55.3 } }, { "modelId": "meta-llama-llama-3.1-8b-instruct", "modelName": "meta-llama/Llama-3.1-8B-Instruct", "provider": "meta-llama", "modelType": "open", "parametersBillions": 8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 54.4, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 30.4, "gsm8k_score": 84.5, "mmluPro_score": 48.3, "aggregate_score": 54.4 } }, { "modelId": "facebook-cwm", "modelName": "facebook/cwm", "provider": "facebook", "modelType": "open", "parametersBillions": 32.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.9, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 53.9, "aggregate_score": 53.9 } }, { "modelId": "minimaxai-minimax-m2.1", "modelName": "MiniMaxAI/MiniMax-M2.1", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.59, "coverageCount": 6, "coveragePercent": 54.5, "scores": { "evasionBench_score": 71.31, "hle_score": 22.2, "mmluPro_score": 88, "swePro_score": 36.81, "sweVerified_score": 74, "terminalBench_score": 29.2, "aggregate_score": 53.59 } }, { "modelId": "redhatai-nvidia-nemotron-3-super-120b-a12b-bf16", "modelName": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "provider": "RedHatAI", "modelType": "open", "parametersBillions": 123.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 53.19, "coverageCount": 5, "coveragePercent": 45.5, "scores": { "gpqa_score": 79.23, "hle_score": 18.26, "mmluPro_score": 83.73, "sweVerified_score": 53.73, "terminalBench_score": 31, "aggregate_score": 53.19 } }, { "modelId": "meituan-longcat-longcat-flash-thinking-2601", "modelName": "meituan-longcat/LongCat-Flash-Thinking-2601", "provider": "meituan-longcat", "modelType": "open", "parametersBillions": 561.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 52.85, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 80.5, "hle_score": 25.2, "aggregate_score": 52.85 } }, { "modelId": "swe-lego-swe-lego-qwen3-32b", "modelName": "SWE-Lego/SWE-Lego-Qwen3-32B", "provider": "SWE-Lego", "modelType": "open", "parametersBillions": 0, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 52.6, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 52.6, "aggregate_score": 52.6 } }, { "modelId": "snurfyai-sn-logicer-0.8b", "modelName": "SnurfyAI/Sn-Logicer-0.8B", "provider": "SnurfyAI", "modelType": "open", "parametersBillions": 0.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 50.57, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gsm8k_score": 50.57, "aggregate_score": 50.57 } }, { "modelId": "qwen-qwen3-coder-next", "modelName": "Qwen/Qwen3-Coder-Next", "provider": "Qwen", "modelType": "open", "parametersBillions": 79.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 50.37, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "swePro_score": 44.3, "sweVerified_score": 70.6, "terminalBench_score": 36.2, "aggregate_score": 50.37 } }, { "modelId": "zai-org-glm-4.7-flash", "modelName": "zai-org/GLM-4.7-Flash", "provider": "zai-org", "modelType": "open", "parametersBillions": 31.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 49.6, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 75.2, "hle_score": 14.4, "sweVerified_score": 59.2, "aggregate_score": 49.6 } }, { "modelId": "minimaxai-minimax-m2", "modelName": "MiniMaxAI/MiniMax-M2", "provider": "MiniMaxAI", "modelType": "open", "parametersBillions": 228.7, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 48.48, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "hle_score": 12.5, "mmluPro_score": 82, "sweVerified_score": 69.4, "terminalBench_score": 30, "aggregate_score": 48.48 } }, { "modelId": "nanbeige-nanbeige4.1-3b", "modelName": "Nanbeige/Nanbeige4.1-3B", "provider": "Nanbeige", "modelType": "open", "parametersBillions": 3.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 48.2, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 83.8, "hle_score": 12.6, "aggregate_score": 48.2 } }, { "modelId": "qwen-qwen3-coder-480b-a35b-instruct", "modelName": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "provider": "Qwen", "modelType": "open", "parametersBillions": 480.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 46.92, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "evasionBench_score": 78.16, "swePro_score": 38.7, "terminalBench_score": 23.9, "aggregate_score": 46.92 } }, { "modelId": "nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16", "modelName": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 46.9, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 15.5, "mmluPro_score": 78.3, "aggregate_score": 46.9 } }, { "modelId": "nvidia-nvidia-nemotron-3-nano-30b-a3b-fp8", "modelName": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "provider": "nvidia", "modelType": "open", "parametersBillions": 31.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 44.15, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 10.2, "mmluPro_score": 78.1, "aggregate_score": 44.15 } }, { "modelId": "swe-lego-swe-lego-qwen3-8b", "modelName": "SWE-Lego/SWE-Lego-Qwen3-8B", "provider": "SWE-Lego", "modelType": "open", "parametersBillions": 0, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 42.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "sweVerified_score": 42.2, "aggregate_score": 42.2 } }, { "modelId": "tiiuae-falcon-h1r-7b", "modelName": "tiiuae/Falcon-H1R-7B", "provider": "tiiuae", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 41.6, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "hle_score": 11.1, "mmluPro_score": 72.1, "aggregate_score": 41.6 } }, { "modelId": "miromind-ai-mirothinker-v1.5-235b", "modelName": "miromind-ai/MiroThinker-v1.5-235B", "provider": "miromind-ai", "modelType": "open", "parametersBillions": 235.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 39.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 39.2, "aggregate_score": 39.2 } }, { "modelId": "liquidai-lfm2.5-1.2b-instruct", "modelName": "LiquidAI/LFM2.5-1.2B-Instruct", "provider": "LiquidAI", "modelType": "open", "parametersBillions": 1.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 38.89, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "gpqa_score": 38.89, "aggregate_score": 38.89 } }, { "modelId": "nvidia-nemotron-orchestrator-8b", "modelName": "nvidia/Nemotron-Orchestrator-8B", "provider": "nvidia", "modelType": "open", "parametersBillions": null, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 37.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 37.1, "aggregate_score": 37.1 } }, { "modelId": "openai-gpt-oss-120b", "modelName": "openai/gpt-oss-120b", "provider": "openai", "modelType": "open", "parametersBillions": 120.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 34.1, "coverageCount": 4, "coveragePercent": 36.4, "scores": { "gpqa_score": 67.1, "hle_score": 5.2, "swePro_score": 16.2, "sweVerified_score": 47.9, "aggregate_score": 34.1 } }, { "modelId": "openai-gpt-oss-20b", "modelName": "openai/gpt-oss-20b", "provider": "openai", "modelType": "open", "parametersBillions": 21.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 32.8, "coverageCount": 3, "coveragePercent": 27.3, "scores": { "gpqa_score": 56.8, "hle_score": 4.2, "sweVerified_score": 37.4, "aggregate_score": 32.8 } }, { "modelId": "miromind-ai-mirothinker-v1.5-30b", "modelName": "miromind-ai/MiroThinker-v1.5-30B", "provider": "miromind-ai", "modelType": "open", "parametersBillions": 30.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 31, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 31, "aggregate_score": 31 } }, { "modelId": "moonshotai-kimi-k2-instruct", "modelName": "moonshotai/Kimi-K2-Instruct", "provider": "moonshotai", "modelType": "open", "parametersBillions": 1026.5, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 27.73, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "swePro_score": 27.67, "terminalBench_score": 27.8, "aggregate_score": 27.73 } }, { "modelId": "nvidia-nemotron-terminal-32b", "modelName": "nvidia/Nemotron-Terminal-32B", "provider": "nvidia", "modelType": "open", "parametersBillions": 32.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 27.4, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "terminalBench_score": 27.4, "aggregate_score": 27.4 } }, { "modelId": "xiaomimimo-mimo-v2-flash", "modelName": "XiaomiMiMo/MiMo-V2-Flash", "provider": "XiaomiMiMo", "modelType": "open", "parametersBillions": 309.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 22.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 22.1, "aggregate_score": 22.1 } }, { "modelId": "qwen-qwen3-235b-a22b", "modelName": "Qwen/Qwen3-235B-A22B", "provider": "Qwen", "modelType": "open", "parametersBillions": 235.1, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 21.41, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "swePro_score": 21.41, "aggregate_score": 21.41 } }, { "modelId": "qwen-qwen3.5-0.8b", "modelName": "Qwen/Qwen3.5-0.8B", "provider": "Qwen", "modelType": "open", "parametersBillions": 0.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 20.8, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "gpqa_score": 11.9, "mmluPro_score": 29.7, "aggregate_score": 20.8 } }, { "modelId": "nvidia-nemotron-terminal-14b", "modelName": "nvidia/Nemotron-Terminal-14B", "provider": "nvidia", "modelType": "open", "parametersBillions": 14.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 20.2, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "terminalBench_score": 20.2, "aggregate_score": 20.2 } }, { "modelId": "openbmb-agentcpm-explore", "modelName": "openbmb/AgentCPM-Explore", "provider": "openbmb", "modelType": "open", "parametersBillions": 4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 19.1, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 19.1, "aggregate_score": 19.1 } }, { "modelId": "zai-org-glm-4.6", "modelName": "zai-org/GLM-4.6", "provider": "zai-org", "modelType": "open", "parametersBillions": 356.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 17.09, "coverageCount": 2, "coveragePercent": 18.2, "scores": { "swePro_score": 9.67, "terminalBench_score": 24.5, "aggregate_score": 17.09 } }, { "modelId": "nvidia-nemotron-terminal-8b", "modelName": "nvidia/Nemotron-Terminal-8B", "provider": "nvidia", "modelType": "open", "parametersBillions": 8.2, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 13, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "terminalBench_score": 13, "aggregate_score": 13 } }, { "modelId": "google-gemma-3-27b-it", "modelName": "google/gemma-3-27b-it", "provider": "google", "modelType": "open", "parametersBillions": 27.4, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 11.38, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "swePro_score": 11.38, "aggregate_score": 11.38 } }, { "modelId": "meta-llama-llama-3.1-405b-instruct", "modelName": "meta-llama/Llama-3.1-405B-Instruct", "provider": "meta-llama", "modelType": "open", "parametersBillions": 405.9, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 11.18, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "swePro_score": 11.18, "aggregate_score": 11.18 } }, { "modelId": "helpingai-dhanishtha-2.0-0126", "modelName": "HelpingAI/Dhanishtha-2.0-0126", "provider": "HelpingAI", "modelType": "open", "parametersBillions": 14.8, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 9.92, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "hle_score": 9.92, "aggregate_score": 9.92 } }, { "modelId": "meta-llama-llama-4-maverick-17b-128e-instruct", "modelName": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "provider": "meta-llama", "modelType": "open", "parametersBillions": 401.6, "license": "Unknown", "contextWindow": 0, "modality": "text", "architecture": "Transformer", "aggregateScore": 5.24, "coverageCount": 1, "coveragePercent": 9.1, "scores": { "swePro_score": 5.24, "aggregate_score": 5.24 } } ] }